Merge pull request #1 from nikhilrayaprolu/new_branch

making a better submission

Dockerfile

+ARG PYTORCH="1.1.0"
+ARG CUDA="10.0"
+ARG CUDNN="7.5"
+
+FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel
+
+RUN apt-get update && apt-get install -y libglib2.0-0 libsm6 libxrender-dev libxext6 \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    bzip2 \
+    cmake \
+    curl \
+    git \
+    g++ \
+    libboost-all-dev \
+    pkg-config \
+    rsync \
+    software-properties-common \
+    sudo \
+    tar \
+    timidity \
+    unzip \
+    wget \
+    locales \
+    zlib1g-dev \
+    python3-dev \
+    python3 \
+    python3-pip \
+    python3-tk \
+    libjpeg-dev \
+    libpng-dev
+
+# Python3
+RUN pip3 install pip --upgrade
+RUN pip3 install utm cython aicrowd_api timeout_decorator \
+  numpy \
+  aicrowd-repo2docker \
+  pillow
+RUN pip3 install git+https://github.com/AIcrowd/coco.git#subdirectory=PythonAPI
+	
+RUN conda install cython -y && conda clean --all
+RUN git clone --branch v1.0rc1 https://github.com/open-mmlab/mmdetection.git /mmdetection
+WORKDIR /mmdetection
+RUN pip install --no-cache-dir -e .
+RUN python3.6 -m pip install aicrowd_api   aicrowd-repo2docker
+# Unicode support:
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+# Enables X11 sharing and creates user home directory
+ENV USER_NAME aicrowd
+ENV HOME_DIR /home/$USER_NAME
+#
+# Replace HOST_UID/HOST_GUID with your user / group id (needed for X11)
+ENV HOST_UID 1000
+ENV HOST_GID 1000
+
+RUN export uid=${HOST_UID} gid=${HOST_GID} && \
+    mkdir -p ${HOME_DIR} && \
+    echo "$USER_NAME:x:${uid}:${gid}:$USER_NAME,,,:$HOME_DIR:/bin/bash" >> /etc/passwd && \
+    echo "$USER_NAME:x:${uid}:" >> /etc/group && \
+    echo "$USER_NAME ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER_NAME && \
+    chmod 0440 /etc/sudoers.d/$USER_NAME && \
+    chown ${uid}:${gid} -R ${HOME_DIR}
+
+USER ${USER_NAME}
+WORKDIR ${HOME_DIR}
+
+COPY . .
+
+RUN sudo chown ${HOST_UID}:${HOST_GID} -R *
+RUN sudo chmod 775 -R *
+
+

aicrowd.json

+{
+    "challenge_id" : "aicrowd-food-recognition-challenge",
+    "grader_id": "aicrowd-food-recognition-challenge",
+    "authors" : ["nikhil13prs"],
+    "description" : "Food Recognition Challenge Submission",
+    "license" : "MIT",
+    "gpu": true
+}
+

aicrowd_helpers.py

+#!/usr/bin/env python
+import aicrowd_api
+import os
+
+########################################################################
+# Instatiate Event Notifier
+########################################################################
+aicrowd_events = aicrowd_api.events.AIcrowdEvents()
+
+
+def execution_start():
+    ########################################################################
+    # Register Evaluation Start event
+    ########################################################################
+    aicrowd_events.register_event(
+                event_type=aicrowd_events.AICROWD_EVENT_INFO,
+                message="execution_started",
+                payload={ #Arbitrary Payload
+                    "event_type": "food_recognition_challenge:execution_started"
+                    }
+                )
+
+
+def execution_progress(progress_payload):
+    image_ids = progress_payload["image_ids"]
+    ########################################################################
+    # Register Evaluation Progress event
+    ########################################################################
+    aicrowd_events.register_event(
+                event_type=aicrowd_events.AICROWD_EVENT_INFO,
+                message="execution_progress",
+                payload={ #Arbitrary Payload
+                    "event_type": "food_recognition_challenge:execution_progress",
+                    "image_ids" : image_ids
+                    }
+                )
+
+def execution_success(payload):
+    predictions_output_path = payload["predictions_output_path"]
+    ########################################################################
+    # Register Evaluation Complete event
+    ########################################################################
+    expected_output_path = os.getenv("AICROWD_PREDICTIONS_OUTPUT_PATH", False)
+    if expected_output_path != predictions_output_path:
+        raise Exception("Please write the output to the path specified in the environment variable : AICROWD_PREDICTIONS_OUTPUT_PATH instead of {}".format(predictions_output_path))
+
+    aicrowd_events.register_event(
+                event_type=aicrowd_events.AICROWD_EVENT_SUCCESS,
+                message="execution_success",
+                payload={ #Arbitrary Payload
+                    "event_type": "food_recognition_challenge:execution_success",
+                    "predictions_output_path" : predictions_output_path
+                    },
+                blocking=True
+                )
+
+def execution_error(error):
+    ########################################################################
+    # Register Evaluation Complete event
+    ########################################################################
+    aicrowd_events.register_event(
+                event_type=aicrowd_events.AICROWD_EVENT_ERROR,
+                message="execution_error",
+                payload={ #Arbitrary Payload
+                    "event_type": "food_recognition_challenge:execution_error",
+                    "error" : error
+                    },
+                blocking=True
+                )
+

configs/htc_r50.py

@@ -181,7 +181,7 @@ test_cfg = dict(
     keep_all_stages=False)
 # dataset settings
 dataset_type = 'CocoDataset'
-data_root = '/ssd_scratch/cvit/mmdetection/data/'
+data_root = '/ssd_scratch/cvit/nikhil/food-version2/data/'
 img_norm_cfg = dict(
     mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
 albu_train_transforms = [
@@ -312,6 +312,6 @@ total_epochs = 30
 dist_params = dict(backend='nccl')
 log_level = 'INFO'
 work_dir = './work_dirs/htc_r50_fpn'
-load_from = '/ssd_scratch/cvit/mmdetection/htc_r50_fpn_20e_20190408-c03b7015.pth' 
+load_from = '/ssd_scratch/cvit/nikhil/food-version2/htc_r50_fpn_20e_20190408-c03b7015.pth' 
 resume_from = None
-workflow = [('train', 1)]
\ No newline at end of file
+workflow = [('train', 1)]

run.sh

+#!/bin/bash
+python tools/test.py configs/htc_r50.py epoch_20.pth --json_out $AICROWD_PREDICTIONS_OUTPUT_PATH
+

test.py

+import argparse
+import os
+import os.path as osp
+import pickle
+import shutil
+import tempfile
+
+import mmcv
+import torch
+import torch.distributed as dist
+from mmcv.parallel import MMDataParallel, MMDistributedDataParallel
+from mmcv.runner import get_dist_info, init_dist, load_checkpoint
+
+from mmdet.core import coco_eval, results2json, wrap_fp16_model
+from mmdet.datasets import build_dataloader, build_dataset
+from mmdet.models import build_detector
+import glob
+import json
+
+test_images_path = os.getenv("AICROWD_TEST_IMAGES_PATH", False)
+predictions_output_path = os.getenv("AICROWD_PREDICTIONS_OUTPUT_PATH", False)
+print(predictions_output_path)
+
+
+
+annotations = {'categories': [], 'info': {}, 'images': []}
+
+
+for item in glob.glob(test_images_path+'/*.jpg'):
+    image_dict = dict()
+    
+    img = mmcv.imread(item)
+    height,width,__ = img.shape
+    id = int(os.path.basename(item).split('.')[0])
+    image_dict['id'] = id
+    image_dict['file_name'] = os.path.basename(item)
+    image_dict['width'] = width
+    image_dict['height'] = height
+    annotations['images'].append(image_dict)
+annotations['categories'] = [
+    {
+      "id": 2578,
+      "name": "water",
+      "name_readable": "Water",
+      "supercategory": "food"
+    },
+    {
+      "id": 2939,
+      "name": "pizza-margherita-baked",
+      "name_readable": "Pizza, Margherita, baked",
+      "supercategory": "food"
+    },
+    {
+      "id": 1085,
+      "name": "broccoli",
+      "name_readable": "Broccoli",
+      "supercategory": "food"
+    },
+    {
+      "id": 1040,
+      "name": "salad-leaf-salad-green",
+      "name_readable": "Salad, leaf / salad, green",
+      "supercategory": "food"
+    },
+    {
+      "id": 1070,
+      "name": "zucchini",
+      "name_readable": "Zucchini",
+      "supercategory": "food"
+    },
+    {
+      "id": 2022,
+      "name": "egg",
+      "name_readable": "Egg",
+      "supercategory": "food"
+    },
+    {
+      "id": 2053,
+      "name": "butter",
+      "name_readable": "Butter",
+      "supercategory": "food"
+    },
+    {
+      "id": 1566,
+      "name": "bread-white",
+      "name_readable": "Bread, white",
+      "supercategory": "food"
+    },
+    {
+      "id": 1151,
+      "name": "apple",
+      "name_readable": "Apple",
+      "supercategory": "food"
+    },
+    {
+      "id": 2131,
+      "name": "dark-chocolate",
+      "name_readable": "Dark chocolate",
+      "supercategory": "food"
+    },
+    {
+      "id": 2521,
+      "name": "white-coffee-with-caffeine",
+      "name_readable": "White coffee, with caffeine",
+      "supercategory": "food"
+    },
+    {
+      "id": 1068,
+      "name": "sweet-pepper",
+      "name_readable": "Sweet pepper",
+      "supercategory": "food"
+    },
+    {
+      "id": 1026,
+      "name": "mixed-salad-chopped-without-sauce",
+      "name_readable": "Mixed salad (chopped without sauce)",
+      "supercategory": "food"
+    },
+    {
+      "id": 2738,
+      "name": "tomato-sauce",
+      "name_readable": "Tomato sauce",
+      "supercategory": "food"
+    },
+    {
+      "id": 1565,
+      "name": "bread-wholemeal",
+      "name_readable": "Bread, wholemeal",
+      "supercategory": "food"
+    },
+    {
+      "id": 2512,
+      "name": "coffee-with-caffeine",
+      "name_readable": "Coffee, with caffeine",
+      "supercategory": "food"
+    },
+    {
+      "id": 1061,
+      "name": "cucumber",
+      "name_readable": "Cucumber",
+      "supercategory": "food"
+    },
+    {
+      "id": 1311,
+      "name": "cheese",
+      "name_readable": "Cheese",
+      "supercategory": "food"
+    },
+    {
+      "id": 1505,
+      "name": "pasta-spaghetti",
+      "name_readable": "Pasta, spaghetti",
+      "supercategory": "food"
+    },
+    {
+      "id": 1468,
+      "name": "rice",
+      "name_readable": "Rice",
+      "supercategory": "food"
+    },
+    {
+      "id": 1967,
+      "name": "salmon",
+      "name_readable": "Salmon",
+      "supercategory": "food"
+    },
+    {
+      "id": 1078,
+      "name": "carrot",
+      "name_readable": "Carrot",
+      "supercategory": "food"
+    },
+    {
+      "id": 1116,
+      "name": "onion",
+      "name_readable": "Onion",
+      "supercategory": "food"
+    },
+    {
+      "id": 1022,
+      "name": "mixed-vegetables",
+      "name_readable": "Mixed vegetables",
+      "supercategory": "food"
+    },
+    {
+      "id": 2504,
+      "name": "espresso-with-caffeine",
+      "name_readable": "Espresso, with caffeine",
+      "supercategory": "food"
+    },
+    {
+      "id": 1154,
+      "name": "banana",
+      "name_readable": "Banana",
+      "supercategory": "food"
+    },
+    {
+      "id": 1163,
+      "name": "strawberries",
+      "name_readable": "Strawberries",
+      "supercategory": "food"
+    },
+    {
+      "id": 2750,
+      "name": "mayonnaise",
+      "name_readable": "Mayonnaise",
+      "supercategory": "food"
+    },
+    {
+      "id": 1210,
+      "name": "almonds",
+      "name_readable": "Almonds",
+      "supercategory": "food"
+    },
+    {
+      "id": 2620,
+      "name": "wine-white",
+      "name_readable": "Wine, white",
+      "supercategory": "food"
+    },
+    {
+      "id": 1310,
+      "name": "hard-cheese",
+      "name_readable": "Hard cheese",
+      "supercategory": "food"
+    },
+    {
+      "id": 1893,
+      "name": "ham-raw",
+      "name_readable": "Ham, raw",
+      "supercategory": "food"
+    },
+    {
+      "id": 1069,
+      "name": "tomato",
+      "name_readable": "Tomato",
+      "supercategory": "food"
+    },
+    {
+      "id": 1058,
+      "name": "french-beans",
+      "name_readable": "French beans",
+      "supercategory": "food"
+    },
+    {
+      "id": 1180,
+      "name": "mandarine",
+      "name_readable": "Mandarine",
+      "supercategory": "food"
+    },
+    {
+      "id": 2618,
+      "name": "wine-red",
+      "name_readable": "Wine, red",
+      "supercategory": "food"
+    },
+    {
+      "id": 1010,
+      "name": "potatoes-steamed",
+      "name_readable": "Potatoes steamed",
+      "supercategory": "food"
+    },
+    {
+      "id": 1588,
+      "name": "croissant",
+      "name_readable": "Croissant",
+      "supercategory": "food"
+    },
+    {
+      "id": 1879,
+      "name": "salami",
+      "name_readable": "Salami",
+      "supercategory": "food"
+    },
+    {
+      "id": 3080,
+      "name": "boisson-au-glucose-50g",
+      "name_readable": "Boisson au glucose 50g",
+      "supercategory": "food"
+    },
+    {
+      "id": 2388,
+      "name": "biscuits",
+      "name_readable": "Biscuits",
+      "supercategory": "food"
+    },
+    {
+      "id": 1108,
+      "name": "corn",
+      "name_readable": "Corn",
+      "supercategory": "food"
+    },
+    {
+      "id": 1032,
+      "name": "leaf-spinach",
+      "name_readable": "Leaf spinach",
+      "supercategory": "food"
+    },
+    {
+      "id": 2099,
+      "name": "jam",
+      "name_readable": "Jam",
+      "supercategory": "food"
+    },
+    {
+      "id": 2530,
+      "name": "tea-green",
+      "name_readable": "Tea, green",
+      "supercategory": "food"
+    },
+    {
+      "id": 1013,
+      "name": "chips-french-fries",
+      "name_readable": "Chips, french fries",
+      "supercategory": "food"
+    },
+    {
+      "id": 1323,
+      "name": "parmesan",
+      "name_readable": "Parmesan",
+      "supercategory": "food"
+    },
+    {
+      "id": 2634,
+      "name": "beer",
+      "name_readable": "Beer",
+      "supercategory": "food"
+    },
+    {
+      "id": 1056,
+      "name": "avocado",
+      "name_readable": "Avocado",
+      "supercategory": "food"
+    },
+    {
+      "id": 1520,
+      "name": "bread-french-white-flour",
+      "name_readable": "Bread, French (white flour)",
+      "supercategory": "food"
+    },
+    {
+      "id": 1788,
+      "name": "chicken",
+      "name_readable": "Chicken",
+      "supercategory": "food"
+    },
+    {
+      "id": 1352,
+      "name": "soft-cheese",
+      "name_readable": "Soft cheese",
+      "supercategory": "food"
+    },
+    {
+      "id": 2498,
+      "name": "tea",
+      "name_readable": "Tea",
+      "supercategory": "food"
+    },
+    {
+      "id": 2711,
+      "name": "sauce-savoury",
+      "name_readable": "Sauce (savoury)",
+      "supercategory": "food"
+    },
+    {
+      "id": 2103,
+      "name": "honey",
+      "name_readable": "Honey",
+      "supercategory": "food"
+    },
+    {
+      "id": 1554,
+      "name": "bread-whole-wheat",
+      "name_readable": "Bread, whole wheat",
+      "supercategory": "food"
+    },
+    {
+      "id": 1556,
+      "name": "bread-sourdough",
+      "name_readable": "Bread, sourdough",
+      "supercategory": "food"
+    },
+    {
+      "id": 1307,
+      "name": "gruyere",
+      "name_readable": "Gruyère",
+      "supercategory": "food"
+    },
+    {
+      "id": 1060,
+      "name": "pickle",
+      "name_readable": "Pickle",
+      "supercategory": "food"
+    },
+    {
+      "id": 1220,
+      "name": "mixed-nuts",
+      "name_readable": "Mixed nuts",
+      "supercategory": "food"
+    },
+    {
+      "id": 2580,
+      "name": "water-mineral",
+      "name_readable": "Water, mineral",
+      "supercategory": "food"
+    }
+  ]
+json.dump(annotations, open('test.json', 'w'))
+
+
+def single_gpu_test(model, data_loader, show=False):
+    model.eval()
+    results = []
+    dataset = data_loader.dataset
+    prog_bar = mmcv.ProgressBar(len(dataset))
+    for i, data in enumerate(data_loader):
+        with torch.no_grad():
+            result = model(return_loss=False, rescale=not show, **data)
+        results.append(result)
+        print("one image done")
+        if show:
+            model.module.show_result(data, result)
+
+        batch_size = data['img'][0].size(0)
+        for _ in range(batch_size):
+            prog_bar.update()
+    return results
+
+
+def multi_gpu_test(model, data_loader, tmpdir=None):
+    model.eval()
+    results = []
+    dataset = data_loader.dataset
+    rank, world_size = get_dist_info()
+    if rank == 0:
+        prog_bar = mmcv.ProgressBar(len(dataset))
+    for i, data in enumerate(data_loader):
+        with torch.no_grad():
+            result = model(return_loss=False, rescale=True, **data)
+        results.append(result)
+
+        if rank == 0:
+            batch_size = data['img'][0].size(0)
+            for _ in range(batch_size * world_size):
+                prog_bar.update()
+
+    # collect results from all ranks
+    results = collect_results(results, len(dataset), tmpdir)
+
+    return results
+
+
+def collect_results(result_part, size,