tests/test_config.py

+from os.path import dirname, exists, join
+
+
+def _get_config_directory():
+    """ Find the predefined detector config directory """
+    try:
+        # Assume we are running in the source mmdetection repo
+        repo_dpath = dirname(dirname(__file__))
+    except NameError:
+        # For IPython development when this __file__ is not defined
+        import mmdet
+        repo_dpath = dirname(dirname(mmdet.__file__))
+    config_dpath = join(repo_dpath, 'configs')
+    if not exists(config_dpath):
+        raise Exception('Cannot find config path')
+    return config_dpath
+
+
+def test_config_build_detector():
+    """
+    Test that all detection models defined in the configs can be initialized.
+    """
+    from xdoctest.utils import import_module_from_path
+    from mmdet.models import build_detector
+
+    config_dpath = _get_config_directory()
+    print('Found config_dpath = {!r}'.format(config_dpath))
+
+    # import glob
+    # config_fpaths = list(glob.glob(join(config_dpath, '**', '*.py')))
+    # config_names = [relpath(p, config_dpath) for p in config_fpaths]
+
+    # Only tests a representative subset of configurations
+
+    config_names = [
+        # 'dcn/faster_rcnn_dconv_c3-c5_r50_fpn_1x.py',
+        # 'dcn/cascade_mask_rcnn_dconv_c3-c5_r50_fpn_1x.py',
+        # 'dcn/faster_rcnn_dpool_r50_fpn_1x.py',
+        'dcn/mask_rcnn_dconv_c3-c5_r50_fpn_1x.py',
+        # 'dcn/faster_rcnn_dconv_c3-c5_x101_32x4d_fpn_1x.py',
+        # 'dcn/cascade_rcnn_dconv_c3-c5_r50_fpn_1x.py',
+        # 'dcn/faster_rcnn_mdpool_r50_fpn_1x.py',
+        # 'dcn/faster_rcnn_mdconv_c3-c5_group4_r50_fpn_1x.py',
+        # 'dcn/faster_rcnn_mdconv_c3-c5_r50_fpn_1x.py',
+        # ---
+        # 'htc/htc_x101_32x4d_fpn_20e_16gpu.py',
+        'htc/htc_without_semantic_r50_fpn_1x.py',
+        # 'htc/htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e.py',
+        # 'htc/htc_x101_64x4d_fpn_20e_16gpu.py',
+        # 'htc/htc_r50_fpn_1x.py',
+        # 'htc/htc_r101_fpn_20e.py',
+        # 'htc/htc_r50_fpn_20e.py',
+        # ---
+        'cityscapes/mask_rcnn_r50_fpn_1x_cityscapes.py',
+        # 'cityscapes/faster_rcnn_r50_fpn_1x_cityscapes.py',
+        # ---
+        # 'scratch/scratch_faster_rcnn_r50_fpn_gn_6x.py',
+        # 'scratch/scratch_mask_rcnn_r50_fpn_gn_6x.py',
+        # ---
+        # 'grid_rcnn/grid_rcnn_gn_head_x101_32x4d_fpn_2x.py',
+        'grid_rcnn/grid_rcnn_gn_head_r50_fpn_2x.py',
+        # ---
+        'double_heads/dh_faster_rcnn_r50_fpn_1x.py',
+        # ---
+        'empirical_attention/faster_rcnn_r50_fpn_attention_0010_dcn_1x.py',
+        # 'empirical_attention/faster_rcnn_r50_fpn_attention_1111_1x.py',
+        # 'empirical_attention/faster_rcnn_r50_fpn_attention_0010_1x.py',
+        # 'empirical_attention/faster_rcnn_r50_fpn_attention_1111_dcn_1x.py',
+        # ---
+        # 'ms_rcnn/ms_rcnn_r101_caffe_fpn_1x.py',
+        # 'ms_rcnn/ms_rcnn_x101_64x4d_fpn_1x.py',
+        # 'ms_rcnn/ms_rcnn_r50_caffe_fpn_1x.py',
+        # ---
+        # 'guided_anchoring/ga_faster_x101_32x4d_fpn_1x.py',
+        # 'guided_anchoring/ga_rpn_x101_32x4d_fpn_1x.py',
+        # 'guided_anchoring/ga_retinanet_r50_caffe_fpn_1x.py',
+        # 'guided_anchoring/ga_fast_r50_caffe_fpn_1x.py',
+        # 'guided_anchoring/ga_retinanet_x101_32x4d_fpn_1x.py',
+        # 'guided_anchoring/ga_rpn_r101_caffe_rpn_1x.py',
+        # 'guided_anchoring/ga_faster_r50_caffe_fpn_1x.py',
+        'guided_anchoring/ga_rpn_r50_caffe_fpn_1x.py',
+        # ---
+        'foveabox/fovea_r50_fpn_4gpu_1x.py',
+        # 'foveabox/fovea_align_gn_ms_r101_fpn_4gpu_2x.py',
+        # 'foveabox/fovea_align_gn_r50_fpn_4gpu_2x.py',
+        # 'foveabox/fovea_align_gn_r101_fpn_4gpu_2x.py',
+        'foveabox/fovea_align_gn_ms_r50_fpn_4gpu_2x.py',
+        # ---
+        # 'hrnet/cascade_rcnn_hrnetv2p_w32_20e.py',
+        # 'hrnet/mask_rcnn_hrnetv2p_w32_1x.py',
+        # 'hrnet/cascade_mask_rcnn_hrnetv2p_w32_20e.py',
+        # 'hrnet/htc_hrnetv2p_w32_20e.py',
+        # 'hrnet/faster_rcnn_hrnetv2p_w18_1x.py',
+        # 'hrnet/mask_rcnn_hrnetv2p_w18_1x.py',
+        # 'hrnet/faster_rcnn_hrnetv2p_w32_1x.py',
+        # 'hrnet/faster_rcnn_hrnetv2p_w40_1x.py',
+        'hrnet/fcos_hrnetv2p_w32_gn_1x_4gpu.py',
+        # ---
+        # 'gn+ws/faster_rcnn_r50_fpn_gn_ws_1x.py',
+        # 'gn+ws/mask_rcnn_x101_32x4d_fpn_gn_ws_2x.py',
+        'gn+ws/mask_rcnn_r50_fpn_gn_ws_2x.py',
+        # 'gn+ws/mask_rcnn_r50_fpn_gn_ws_20_23_24e.py',
+        # ---
+        # 'wider_face/ssd300_wider_face.py',
+        # ---
+        'pascal_voc/ssd300_voc.py',
+        'pascal_voc/faster_rcnn_r50_fpn_1x_voc0712.py',
+        'pascal_voc/ssd512_voc.py',
+        # ---
+        # 'gcnet/mask_rcnn_r4_gcb_c3-c5_r50_fpn_syncbn_1x.py',
+        # 'gcnet/mask_rcnn_r16_gcb_c3-c5_r50_fpn_syncbn_1x.py',
+        # 'gcnet/mask_rcnn_r4_gcb_c3-c5_r50_fpn_1x.py',
+        # 'gcnet/mask_rcnn_r16_gcb_c3-c5_r50_fpn_1x.py',
+        'gcnet/mask_rcnn_r50_fpn_sbn_1x.py',
+        # ---
+        'gn/mask_rcnn_r50_fpn_gn_contrib_2x.py',
+        # 'gn/mask_rcnn_r50_fpn_gn_2x.py',
+        # 'gn/mask_rcnn_r101_fpn_gn_2x.py',
+        # ---
+        # 'reppoints/reppoints_moment_x101_dcn_fpn_2x.py',
+        'reppoints/reppoints_moment_r50_fpn_2x.py',
+        # 'reppoints/reppoints_moment_x101_dcn_fpn_2x_mt.py',
+        'reppoints/reppoints_partial_minmax_r50_fpn_1x.py',
+        'reppoints/bbox_r50_grid_center_fpn_1x.py',
+        # 'reppoints/reppoints_moment_r101_dcn_fpn_2x.py',
+        # 'reppoints/reppoints_moment_r101_fpn_2x_mt.py',
+        # 'reppoints/reppoints_moment_r50_fpn_2x_mt.py',
+        'reppoints/reppoints_minmax_r50_fpn_1x.py',
+        # 'reppoints/reppoints_moment_r50_fpn_1x.py',
+        # 'reppoints/reppoints_moment_r101_fpn_2x.py',
+        # 'reppoints/reppoints_moment_r101_dcn_fpn_2x_mt.py',
+        'reppoints/bbox_r50_grid_fpn_1x.py',
+        # ---
+        # 'fcos/fcos_mstrain_640_800_x101_64x4d_fpn_gn_2x.py',
+        # 'fcos/fcos_mstrain_640_800_r101_caffe_fpn_gn_2x_4gpu.py',
+        'fcos/fcos_r50_caffe_fpn_gn_1x_4gpu.py',
+        # ---
+        'albu_example/mask_rcnn_r50_fpn_1x.py',
+        # ---
+        'libra_rcnn/libra_faster_rcnn_r50_fpn_1x.py',
+        # 'libra_rcnn/libra_retinanet_r50_fpn_1x.py',
+        # 'libra_rcnn/libra_faster_rcnn_r101_fpn_1x.py',
+        # 'libra_rcnn/libra_faster_rcnn_x101_64x4d_fpn_1x.py',
+        # 'libra_rcnn/libra_fast_rcnn_r50_fpn_1x.py',
+        # ---
+        # 'ghm/retinanet_ghm_r50_fpn_1x.py',
+        # ---
+        # 'fp16/retinanet_r50_fpn_fp16_1x.py',
+        'fp16/mask_rcnn_r50_fpn_fp16_1x.py',
+        'fp16/faster_rcnn_r50_fpn_fp16_1x.py'
+    ]
+
+    print('Using {} config files'.format(len(config_names)))
+
+    for config_fname in config_names:
+        config_fpath = join(config_dpath, config_fname)
+        config_mod = import_module_from_path(config_fpath)
+
+        config_mod.model
+        config_mod.train_cfg
+        config_mod.test_cfg
+        print('Building detector, config_fpath = {!r}'.format(config_fpath))
+
+        # Remove pretrained keys to allow for testing in an offline environment
+        if 'pretrained' in config_mod.model:
+            config_mod.model['pretrained'] = None
+
+        detector = build_detector(
+            config_mod.model,
+            train_cfg=config_mod.train_cfg,
+            test_cfg=config_mod.test_cfg)
+        assert detector is not None

tests/test_forward.py

+"""
+pytest tests/test_forward.py
+"""
+import copy
+from os.path import dirname, exists, join
+
+import numpy as np
+import torch
+
+
+def _get_config_directory():
+    """ Find the predefined detector config directory """
+    try:
+        # Assume we are running in the source mmdetection repo
+        repo_dpath = dirname(dirname(__file__))
+    except NameError:
+        # For IPython development when this __file__ is not defined
+        import mmdet
+        repo_dpath = dirname(dirname(mmdet.__file__))
+    config_dpath = join(repo_dpath, 'configs')
+    if not exists(config_dpath):
+        raise Exception('Cannot find config path')
+    return config_dpath
+
+
+def _get_config_module(fname):
+    """
+    Load a configuration as a python module
+    """
+    from xdoctest.utils import import_module_from_path
+    config_dpath = _get_config_directory()
+    config_fpath = join(config_dpath, fname)
+    config_mod = import_module_from_path(config_fpath)
+    return config_mod
+
+
+def _get_detector_cfg(fname):
+    """
+    Grab configs necessary to create a detector. These are deep copied to allow
+    for safe modification of parameters without influencing other tests.
+    """
+    import mmcv
+    config = _get_config_module(fname)
+    model = copy.deepcopy(config.model)
+    train_cfg = mmcv.Config(copy.deepcopy(config.train_cfg))
+    test_cfg = mmcv.Config(copy.deepcopy(config.test_cfg))
+    return model, train_cfg, test_cfg
+
+
+def test_ssd300_forward():
+    model, train_cfg, test_cfg = _get_detector_cfg('ssd300_coco.py')
+    model['pretrained'] = None
+
+    from mmdet.models import build_detector
+    detector = build_detector(model, train_cfg=train_cfg, test_cfg=test_cfg)
+
+    input_shape = (1, 3, 300, 300)
+    mm_inputs = _demo_mm_inputs(input_shape)
+
+    imgs = mm_inputs.pop('imgs')
+    img_metas = mm_inputs.pop('img_metas')
+
+    # Test forward train
+    gt_bboxes = mm_inputs['gt_bboxes']
+    gt_labels = mm_inputs['gt_labels']
+    losses = detector.forward(
+        imgs,
+        img_metas,
+        gt_bboxes=gt_bboxes,
+        gt_labels=gt_labels,
+        return_loss=True)
+    assert isinstance(losses, dict)
+
+    # Test forward test
+    with torch.no_grad():
+        img_list = [g[None, :] for g in imgs]
+        batch_results = []
+        for one_img, one_meta in zip(img_list, img_metas):
+            result = detector.forward([one_img], [[one_meta]],
+                                      return_loss=False)
+            batch_results.append(result)
+
+
+def test_rpn_forward():
+    model, train_cfg, test_cfg = _get_detector_cfg('rpn_r50_fpn_1x.py')
+    model['pretrained'] = None
+
+    from mmdet.models import build_detector
+    detector = build_detector(model, train_cfg=train_cfg, test_cfg=test_cfg)
+
+    input_shape = (1, 3, 224, 224)
+    mm_inputs = _demo_mm_inputs(input_shape)
+
+    imgs = mm_inputs.pop('imgs')
+    img_metas = mm_inputs.pop('img_metas')
+
+    # Test forward train
+    gt_bboxes = mm_inputs['gt_bboxes']
+    losses = detector.forward(
+        imgs, img_metas, gt_bboxes=gt_bboxes, return_loss=True)
+    assert isinstance(losses, dict)
+
+    # Test forward test
+    with torch.no_grad():
+        img_list = [g[None, :] for g in imgs]
+        batch_results = []
+        for one_img, one_meta in zip(img_list, img_metas):
+            result = detector.forward([one_img], [[one_meta]],
+                                      return_loss=False)
+            batch_results.append(result)
+
+
+def test_retina_ghm_forward():
+    model, train_cfg, test_cfg = _get_detector_cfg(
+        'ghm/retinanet_ghm_r50_fpn_1x.py')
+    model['pretrained'] = None
+
+    from mmdet.models import build_detector
+    detector = build_detector(model, train_cfg=train_cfg, test_cfg=test_cfg)
+
+    input_shape = (3, 3, 224, 224)
+    mm_inputs = _demo_mm_inputs(input_shape)
+
+    imgs = mm_inputs.pop('imgs')
+    img_metas = mm_inputs.pop('img_metas')
+
+    # Test forward train
+    gt_bboxes = mm_inputs['gt_bboxes']
+    gt_labels = mm_inputs['gt_labels']
+    losses = detector.forward(
+        imgs,
+        img_metas,
+        gt_bboxes=gt_bboxes,
+        gt_labels=gt_labels,
+        return_loss=True)
+    assert isinstance(losses, dict)
+
+    # Test forward test
+    with torch.no_grad():
+        img_list = [g[None, :] for g in imgs]
+        batch_results = []
+        for one_img, one_meta in zip(img_list, img_metas):
+            result = detector.forward([one_img], [[one_meta]],
+                                      return_loss=False)
+            batch_results.append(result)
+
+    if torch.cuda.is_available():
+        detector = detector.cuda()
+        imgs = imgs.cuda()
+        # Test forward train
+        gt_bboxes = [b.cuda() for b in mm_inputs['gt_bboxes']]
+        gt_labels = [g.cuda() for g in mm_inputs['gt_labels']]
+        losses = detector.forward(
+            imgs,
+            img_metas,
+            gt_bboxes=gt_bboxes,
+            gt_labels=gt_labels,
+            return_loss=True)
+        assert isinstance(losses, dict)
+
+        # Test forward test
+        with torch.no_grad():
+            img_list = [g[None, :] for g in imgs]
+            batch_results = []
+            for one_img, one_meta in zip(img_list, img_metas):
+                result = detector.forward([one_img], [[one_meta]],
+                                          return_loss=False)
+                batch_results.append(result)
+
+
+def test_cascade_forward():
+    try:
+        from torchvision import _C as C  # NOQA
+    except ImportError:
+        import pytest
+        raise pytest.skip('requires torchvision on cpu')
+
+    model, train_cfg, test_cfg = _get_detector_cfg(
+        'cascade_rcnn_r50_fpn_1x.py')
+    model['pretrained'] = None
+    # torchvision roi align supports CPU
+    model['bbox_roi_extractor']['roi_layer']['use_torchvision'] = True
+
+    from mmdet.models import build_detector
+    detector = build_detector(model, train_cfg=train_cfg, test_cfg=test_cfg)
+
+    input_shape = (1, 3, 256, 256)
+
+    # Test forward train with a non-empty truth batch
+    mm_inputs = _demo_mm_inputs(input_shape, num_items=[10])
+    imgs = mm_inputs.pop('imgs')
+    img_metas = mm_inputs.pop('img_metas')
+    gt_bboxes = mm_inputs['gt_bboxes']
+    gt_labels = mm_inputs['gt_labels']
+    losses = detector.forward(
+        imgs,
+        img_metas,
+        gt_bboxes=gt_bboxes,
+        gt_labels=gt_labels,
+        return_loss=True)
+    assert isinstance(losses, dict)
+    from mmdet.apis.train import parse_losses
+    total_loss = float(parse_losses(losses)[0].item())
+    assert total_loss > 0
+
+    # Test forward train with an empty truth batch
+    mm_inputs = _demo_mm_inputs(input_shape, num_items=[0])
+    imgs = mm_inputs.pop('imgs')
+    img_metas = mm_inputs.pop('img_metas')
+    gt_bboxes = mm_inputs['gt_bboxes']
+    gt_labels = mm_inputs['gt_labels']
+    losses = detector.forward(
+        imgs,
+        img_metas,
+        gt_bboxes=gt_bboxes,
+        gt_labels=gt_labels,
+        return_loss=True)
+    assert isinstance(losses, dict)
+    from mmdet.apis.train import parse_losses
+    total_loss = float(parse_losses(losses)[0].item())
+    assert total_loss > 0
+
+
+def test_faster_rcnn_forward():
+    try:
+        from torchvision import _C as C  # NOQA
+    except ImportError:
+        import pytest
+        raise pytest.skip('requires torchvision on cpu')
+
+    model, train_cfg, test_cfg = _get_detector_cfg('faster_rcnn_r50_fpn_1x.py')
+    model['pretrained'] = None
+    # torchvision roi align supports CPU
+    model['bbox_roi_extractor']['roi_layer']['use_torchvision'] = True
+
+    from mmdet.models import build_detector
+    detector = build_detector(model, train_cfg=train_cfg, test_cfg=test_cfg)
+
+    input_shape = (1, 3, 256, 256)
+
+    # Test forward train with a non-empty truth batch
+    mm_inputs = _demo_mm_inputs(input_shape, num_items=[10])
+    imgs = mm_inputs.pop('imgs')
+    img_metas = mm_inputs.pop('img_metas')
+    gt_bboxes = mm_inputs['gt_bboxes']
+    gt_labels = mm_inputs['gt_labels']
+    losses = detector.forward(
+        imgs,
+        img_metas,
+        gt_bboxes=gt_bboxes,
+        gt_labels=gt_labels,
+        return_loss=True)
+    assert isinstance(losses, dict)
+    from mmdet.apis.train import parse_losses
+    total_loss = float(parse_losses(losses)[0].item())
+    assert total_loss > 0
+
+    # Test forward train with an empty truth batch
+    mm_inputs = _demo_mm_inputs(input_shape, num_items=[0])
+    imgs = mm_inputs.pop('imgs')
+    img_metas = mm_inputs.pop('img_metas')
+    gt_bboxes = mm_inputs['gt_bboxes']
+    gt_labels = mm_inputs['gt_labels']
+    losses = detector.forward(
+        imgs,
+        img_metas,
+        gt_bboxes=gt_bboxes,
+        gt_labels=gt_labels,
+        return_loss=True)
+    assert isinstance(losses, dict)
+    from mmdet.apis.train import parse_losses
+    total_loss = float(parse_losses(losses)[0].item())
+    assert total_loss > 0
+
+
+def test_faster_rcnn_ohem_forward():
+    try:
+        from torchvision import _C as C  # NOQA
+    except ImportError:
+        import pytest
+        raise pytest.skip('requires torchvision on cpu')
+
+    model, train_cfg, test_cfg = _get_detector_cfg(
+        'faster_rcnn_ohem_r50_fpn_1x.py')
+    model['pretrained'] = None
+    # torchvision roi align supports CPU
+    model['bbox_roi_extractor']['roi_layer']['use_torchvision'] = True
+
+    from mmdet.models import build_detector
+    detector = build_detector(model, train_cfg=train_cfg, test_cfg=test_cfg)
+
+    input_shape = (1, 3, 256, 256)
+
+    # Test forward train with a non-empty truth batch
+    mm_inputs = _demo_mm_inputs(input_shape, num_items=[10])
+    imgs = mm_inputs.pop('imgs')
+    img_metas = mm_inputs.pop('img_metas')
+    gt_bboxes = mm_inputs['gt_bboxes']
+    gt_labels = mm_inputs['gt_labels']
+    losses = detector.forward(
+        imgs,
+        img_metas,
+        gt_bboxes=gt_bboxes,
+        gt_labels=gt_labels,
+        return_loss=True)
+    assert isinstance(losses, dict)
+    from mmdet.apis.train import parse_losses
+    total_loss = float(parse_losses(losses)[0].item())
+    assert total_loss > 0
+
+    # Test forward train with an empty truth batch
+    mm_inputs = _demo_mm_inputs(input_shape, num_items=[0])
+    imgs = mm_inputs.pop('imgs')
+    img_metas = mm_inputs.pop('img_metas')
+    gt_bboxes = mm_inputs['gt_bboxes']
+    gt_labels = mm_inputs['gt_labels']
+    losses = detector.forward(
+        imgs,
+        img_metas,
+        gt_bboxes=gt_bboxes,
+        gt_labels=gt_labels,
+        return_loss=True)
+    assert isinstance(losses, dict)
+    from mmdet.apis.train import parse_losses
+    total_loss = float(parse_losses(losses)[0].item())
+    assert total_loss > 0
+
+
+def _demo_mm_inputs(input_shape=(1, 3, 300, 300),
+                    num_items=None, num_classes=10):  # yapf: disable
+    """
+    Create a superset of inputs needed to run test or train batches.
+
+    Args:
+        input_shape (tuple):
+            input batch dimensions
+
+        num_items (None | List[int]):
+            specifies the number of boxes in each batch item
+
+        num_classes (int):
+            number of different labels a box might have
+    """
+    (N, C, H, W) = input_shape
+
+    rng = np.random.RandomState(0)
+
+    imgs = rng.rand(*input_shape)
+
+    img_metas = [{
+        'img_shape': (H, W, C),
+        'ori_shape': (H, W, C),
+        'pad_shape': (H, W, C),
+        'filename': '<demo>.png',
+        'scale_factor': 1.0,
+        'flip': False,
+    } for _ in range(N)]
+
+    gt_bboxes = []
+    gt_labels = []
+
+    for batch_idx in range(N):
+        if num_items is None:
+            num_boxes = rng.randint(1, 10)
+        else:
+            num_boxes = num_items[batch_idx]
+
+        cx, cy, bw, bh = rng.rand(num_boxes, 4).T
+
+        tl_x = ((cx * W) - (W * bw / 2)).clip(0, W)
+        tl_y = ((cy * H) - (H * bh / 2)).clip(0, H)
+        br_x = ((cx * W) + (W * bw / 2)).clip(0, W)
+        br_y = ((cy * H) + (H * bh / 2)).clip(0, H)
+
+        boxes = np.vstack([tl_x, tl_y, br_x, br_y]).T
+        class_idxs = rng.randint(1, num_classes, size=num_boxes)
+
+        gt_bboxes.append(torch.FloatTensor(boxes))
+        gt_labels.append(torch.LongTensor(class_idxs))
+
+    mm_inputs = {
+        'imgs': torch.FloatTensor(imgs),
+        'img_metas': img_metas,
+        'gt_bboxes': gt_bboxes,
+        'gt_labels': gt_labels,
+        'gt_bboxes_ignore': None,
+    }
+    return mm_inputs

tests/test_heads.py

+import mmcv
+import torch
+
+from mmdet.core import build_assigner, build_sampler
+from mmdet.models.anchor_heads import AnchorHead
+from mmdet.models.bbox_heads import BBoxHead
+
+
+def test_anchor_head_loss():
+    """
+    Tests anchor head loss when truth is empty and non-empty
+    """
+    self = AnchorHead(num_classes=4, in_channels=1)
+    s = 256
+    img_metas = [{
+        'img_shape': (s, s, 3),
+        'scale_factor': 1,
+        'pad_shape': (s, s, 3)
+    }]
+
+    cfg = mmcv.Config({
+        'assigner': {
+            'type': 'MaxIoUAssigner',
+            'pos_iou_thr': 0.7,
+            'neg_iou_thr': 0.3,
+            'min_pos_iou': 0.3,
+            'ignore_iof_thr': -1
+        },
+        'sampler': {
+            'type': 'RandomSampler',
+            'num': 256,
+            'pos_fraction': 0.5,
+            'neg_pos_ub': -1,
+            'add_gt_as_proposals': False
+        },
+        'allowed_border': 0,
+        'pos_weight': -1,
+        'debug': False
+    })
+
+    # Anchor head expects a multiple levels of features per image
+    feat = [
+        torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2)))
+        for i in range(len(self.anchor_generators))
+    ]
+    cls_scores, bbox_preds = self.forward(feat)
+
+    # Test that empty ground truth encourages the network to predict background
+    gt_bboxes = [torch.empty((0, 4))]
+    gt_labels = [torch.LongTensor([])]
+
+    gt_bboxes_ignore = None
+    empty_gt_losses = self.loss(cls_scores, bbox_preds, gt_bboxes, gt_labels,
+                                img_metas, cfg, gt_bboxes_ignore)
+    # When there is no truth, the cls loss should be nonzero but there should
+    # be no box loss.
+    empty_cls_loss = sum(empty_gt_losses['loss_cls'])
+    empty_box_loss = sum(empty_gt_losses['loss_bbox'])
+    assert empty_cls_loss.item() > 0, 'cls loss should be non-zero'
+    assert empty_box_loss.item() == 0, (
+        'there should be no box loss when there are no true boxes')
+
+    # When truth is non-empty then both cls and box loss should be nonzero for
+    # random inputs
+    gt_bboxes = [
+        torch.Tensor([[23.6667, 23.8757, 238.6326, 151.8874]]),
+    ]
+    gt_labels = [torch.LongTensor([2])]
+    one_gt_losses = self.loss(cls_scores, bbox_preds, gt_bboxes, gt_labels,
+                              img_metas, cfg, gt_bboxes_ignore)
+    onegt_cls_loss = sum(one_gt_losses['loss_cls'])
+    onegt_box_loss = sum(one_gt_losses['loss_bbox'])
+    assert onegt_cls_loss.item() > 0, 'cls loss should be non-zero'
+    assert onegt_box_loss.item() > 0, 'box loss should be non-zero'
+
+
+def test_bbox_head_loss():
+    """
+    Tests bbox head loss when truth is empty and non-empty
+    """
+    self = BBoxHead(in_channels=8, roi_feat_size=3)
+
+    num_imgs = 1
+    feat = torch.rand(1, 1, 3, 3)
+
+    # Dummy proposals
+    proposal_list = [
+        torch.Tensor([[23.6667, 23.8757, 228.6326, 153.8874]]),
+    ]
+
+    target_cfg = mmcv.Config({'pos_weight': 1})
+
+    def _dummy_bbox_sampling(proposal_list, gt_bboxes, gt_labels):
+        """
+        Create sample results that can be passed to BBoxHead.get_target
+        """
+        assign_config = {
+            'type': 'MaxIoUAssigner',
+            'pos_iou_thr': 0.5,
+            'neg_iou_thr': 0.5,
+            'min_pos_iou': 0.5,
+            'ignore_iof_thr': -1
+        }
+        sampler_config = {
+            'type': 'RandomSampler',
+            'num': 512,
+            'pos_fraction': 0.25,
+            'neg_pos_ub': -1,
+            'add_gt_as_proposals': True
+        }
+        bbox_assigner = build_assigner(assign_config)
+        bbox_sampler = build_sampler(sampler_config)
+        gt_bboxes_ignore = [None for _ in range(num_imgs)]
+        sampling_results = []
+        for i in range(num_imgs):
+            assign_result = bbox_assigner.assign(proposal_list[i],
+                                                 gt_bboxes[i],
+                                                 gt_bboxes_ignore[i],
+                                                 gt_labels[i])
+            sampling_result = bbox_sampler.sample(
+                assign_result,
+                proposal_list[i],
+                gt_bboxes[i],
+                gt_labels[i],
+                feats=feat)
+            sampling_results.append(sampling_result)
+        return sampling_results
+
+    # Test bbox loss when truth is empty
+    gt_bboxes = [torch.empty((0, 4))]
+    gt_labels = [torch.LongTensor([])]
+
+    sampling_results = _dummy_bbox_sampling(proposal_list, gt_bboxes,
+                                            gt_labels)
+
+    bbox_targets = self.get_target(sampling_results, gt_bboxes, gt_labels,
+                                   target_cfg)
+    labels, label_weights, bbox_targets, bbox_weights = bbox_targets
+
+    # Create dummy features "extracted" for each sampled bbox
+    num_sampled = sum(len(res.bboxes) for res in sampling_results)
+    dummy_feats = torch.rand(num_sampled, 8 * 3 * 3)
+    cls_scores, bbox_preds = self.forward(dummy_feats)
+
+    losses = self.loss(cls_scores, bbox_preds, labels, label_weights,
+                       bbox_targets, bbox_weights)
+    assert losses.get('loss_cls', 0) > 0, 'cls-loss should be non-zero'
+    assert losses.get('loss_bbox', 0) == 0, 'empty gt loss should be zero'
+
+    # Test bbox loss when truth is non-empty
+    gt_bboxes = [
+        torch.Tensor([[23.6667, 23.8757, 238.6326, 151.8874]]),
+    ]
+    gt_labels = [torch.LongTensor([2])]
+
+    sampling_results = _dummy_bbox_sampling(proposal_list, gt_bboxes,
+                                            gt_labels)
+
+    bbox_targets = self.get_target(sampling_results, gt_bboxes, gt_labels,
+                                   target_cfg)
+    labels, label_weights, bbox_targets, bbox_weights = bbox_targets
+
+    # Create dummy features "extracted" for each sampled bbox
+    num_sampled = sum(len(res.bboxes) for res in sampling_results)
+    dummy_feats = torch.rand(num_sampled, 8 * 3 * 3)
+    cls_scores, bbox_preds = self.forward(dummy_feats)
+
+    losses = self.loss(cls_scores, bbox_preds, labels, label_weights,
+                       bbox_targets, bbox_weights)
+    assert losses.get('loss_cls', 0) > 0, 'cls-loss should be non-zero'
+    assert losses.get('loss_bbox', 0) > 0, 'box-loss should be non-zero'
+
+
+def test_refine_boxes():
+    """
+    Mirrors the doctest in
+    ``mmdet.models.bbox_heads.bbox_head.BBoxHead.refine_boxes`` but checks for
+    multiple values of n_roi / n_img.
+    """
+    self = BBoxHead(reg_class_agnostic=True)
+
+    test_settings = [
+
+        # Corner case: less rois than images
+        {
+            'n_roi': 2,
+            'n_img': 4,
+            'rng': 34285940
+        },
+
+        # Corner case: no images
+        {
+            'n_roi': 0,
+            'n_img': 0,
+            'rng': 52925222
+        },
+
+        # Corner cases: few images / rois
+        {
+            'n_roi': 1,
+            'n_img': 1,
+            'rng': 1200281
+        },
+        {
+            'n_roi': 2,
+            'n_img': 1,
+            'rng': 1200282
+        },
+        {
+            'n_roi': 2,
+            'n_img': 2,
+            'rng': 1200283
+        },
+        {
+            'n_roi': 1,
+            'n_img': 2,
+            'rng': 1200284
+        },
+
+        # Corner case: no rois few images
+        {
+            'n_roi': 0,
+            'n_img': 1,
+            'rng': 23955860
+        },
+        {
+            'n_roi': 0,
+            'n_img': 2,
+            'rng': 25830516
+        },
+
+        # Corner case: no rois many images
+        {
+            'n_roi': 0,
+            'n_img': 10,
+            'rng': 671346
+        },
+        {
+            'n_roi': 0,
+            'n_img': 20,
+            'rng': 699807
+        },
+
+        # Corner case: similar num rois and images
+        {
+            'n_roi': 20,
+            'n_img': 20,
+            'rng': 1200238
+        },
+        {
+            'n_roi': 10,
+            'n_img': 20,
+            'rng': 1200238
+        },
+        {
+            'n_roi': 5,
+            'n_img': 5,
+            'rng': 1200238
+        },
+
+        # ----------------------------------
+        # Common case: more rois than images
+        {
+            'n_roi': 100,
+            'n_img': 1,
+            'rng': 337156
+        },
+        {
+            'n_roi': 150,
+            'n_img': 2,
+            'rng': 275898
+        },
+        {
+            'n_roi': 500,
+            'n_img': 5,
+            'rng': 4903221
+        },
+    ]
+
+    for demokw in test_settings:
+        try:
+            n_roi = demokw['n_roi']
+            n_img = demokw['n_img']
+            rng = demokw['rng']
+
+            print('Test refine_boxes case: {!r}'.format(demokw))
+            tup = _demodata_refine_boxes(n_roi, n_img, rng=rng)
+            rois, labels, bbox_preds, pos_is_gts, img_metas = tup
+            bboxes_list = self.refine_bboxes(rois, labels, bbox_preds,
+                                             pos_is_gts, img_metas)
+            assert len(bboxes_list) == n_img
+            assert sum(map(len, bboxes_list)) <= n_roi
+            assert all(b.shape[1] == 4 for b in bboxes_list)
+        except Exception:
+            print('Test failed with demokw={!r}'.format(demokw))
+            raise
+
+
+def _demodata_refine_boxes(n_roi, n_img, rng=0):
+    """
+    Create random test data for the
+    ``mmdet.models.bbox_heads.bbox_head.BBoxHead.refine_boxes`` method
+    """
+    import numpy as np
+    from mmdet.core.bbox.demodata import random_boxes
+    from mmdet.core.bbox.demodata import ensure_rng
+    try:
+        import kwarray
+    except ImportError:
+        import pytest
+        pytest.skip('kwarray is required for this test')
+    scale = 512
+    rng = ensure_rng(rng)
+    img_metas = [{'img_shape': (scale, scale)} for _ in range(n_img)]
+    # Create rois in the expected format
+    roi_boxes = random_boxes(n_roi, scale=scale, rng=rng)
+    if n_img == 0:
+        assert n_roi == 0, 'cannot have any rois if there are no images'
+        img_ids = torch.empty((0, ), dtype=torch.long)
+        roi_boxes = torch.empty((0, 4), dtype=torch.float32)
+    else:
+        img_ids = rng.randint(0, n_img, (n_roi, ))
+        img_ids = torch.from_numpy(img_ids)
+    rois = torch.cat([img_ids[:, None].float(), roi_boxes], dim=1)
+    # Create other args
+    labels = rng.randint(0, 2, (n_roi, ))
+    labels = torch.from_numpy(labels).long()
+    bbox_preds = random_boxes(n_roi, scale=scale, rng=rng)
+    # For each image, pretend random positive boxes are gts
+    is_label_pos = (labels.numpy() > 0).astype(np.int)
+    lbl_per_img = kwarray.group_items(is_label_pos, img_ids.numpy())
+    pos_per_img = [sum(lbl_per_img.get(gid, [])) for gid in range(n_img)]
+    # randomly generate with numpy then sort with torch
+    _pos_is_gts = [
+        rng.randint(0, 2, (npos, )).astype(np.uint8) for npos in pos_per_img
+    ]
+    pos_is_gts = [
+        torch.from_numpy(p).sort(descending=True)[0] for p in _pos_is_gts
+    ]
+    return rois, labels, bbox_preds, pos_is_gts, img_metas

tests/test_nms.py

+"""
+CommandLine:
+    pytest tests/test_nms.py
+"""
+import numpy as np
+import torch
+
+from mmdet.ops.nms.nms_wrapper import nms
+
+
+def test_nms_device_and_dtypes_cpu():
+    """
+    CommandLine:
+        xdoctest -m tests/test_nms.py test_nms_device_and_dtypes_cpu
+    """
+    iou_thr = 0.7
+    base_dets = np.array([[49.1, 32.4, 51.0, 35.9, 0.9],
+                          [49.3, 32.9, 51.0, 35.3, 0.9],
+                          [35.3, 11.5, 39.9, 14.5, 0.4],
+                          [35.2, 11.7, 39.7, 15.7, 0.3]])
+
+    # CPU can handle float32 and float64
+    dets = base_dets.astype(np.float32)
+    supressed, inds = nms(dets, iou_thr)
+    assert dets.dtype == supressed.dtype
+    assert len(inds) == len(supressed) == 3
+
+    dets = torch.FloatTensor(base_dets)
+    surpressed, inds = nms(dets, iou_thr)
+    assert dets.dtype == surpressed.dtype
+    assert len(inds) == len(surpressed) == 3
+
+    dets = base_dets.astype(np.float64)
+    supressed, inds = nms(dets, iou_thr)
+    assert dets.dtype == supressed.dtype
+    assert len(inds) == len(supressed) == 3
+
+    dets = torch.DoubleTensor(base_dets)
+    surpressed, inds = nms(dets, iou_thr)
+    assert dets.dtype == surpressed.dtype
+    assert len(inds) == len(surpressed) == 3
+
+
+def test_nms_device_and_dtypes_gpu():
+    """
+    CommandLine:
+        xdoctest -m tests/test_nms.py test_nms_device_and_dtypes_gpu
+    """
+    if not torch.cuda.is_available():
+        import pytest
+        pytest.skip('test requires GPU and torch+cuda')
+
+    iou_thr = 0.7
+    base_dets = np.array([[49.1, 32.4, 51.0, 35.9, 0.9],
+                          [49.3, 32.9, 51.0, 35.3, 0.9],
+                          [35.3, 11.5, 39.9, 14.5, 0.4],
+                          [35.2, 11.7, 39.7, 15.7, 0.3]])
+
+    for device_id in range(torch.cuda.device_count()):
+        print('Run NMS on device_id = {!r}'.format(device_id))
+        # GPU can handle float32 but not float64
+        dets = base_dets.astype(np.float32)
+        supressed, inds = nms(dets, iou_thr, device_id)
+        assert dets.dtype == supressed.dtype
+        assert len(inds) == len(supressed) == 3
+
+        dets = torch.FloatTensor(base_dets).to(device_id)
+        surpressed, inds = nms(dets, iou_thr)
+        assert dets.dtype == surpressed.dtype
+        assert len(inds) == len(surpressed) == 3

tests/test_sampler.py

+import torch
+
+from mmdet.core import MaxIoUAssigner
+from mmdet.core.bbox.samplers import OHEMSampler, RandomSampler
+
+
+def test_random_sampler():
+    assigner = MaxIoUAssigner(
+        pos_iou_thr=0.5,
+        neg_iou_thr=0.5,
+        ignore_iof_thr=0.5,
+        ignore_wrt_candidates=False,
+    )
+    bboxes = torch.FloatTensor([
+        [0, 0, 10, 10],
+        [10, 10, 20, 20],
+        [5, 5, 15, 15],
+        [32, 32, 38, 42],
+    ])
+    gt_bboxes = torch.FloatTensor([
+        [0, 0, 10, 9],
+        [0, 10, 10, 19],
+    ])
+    gt_labels = torch.LongTensor([1, 2])
+    gt_bboxes_ignore = torch.Tensor([
+        [30, 30, 40, 40],
+    ])
+    assign_result = assigner.assign(
+        bboxes,
+        gt_bboxes,
+        gt_bboxes_ignore=gt_bboxes_ignore,
+        gt_labels=gt_labels)
+
+    sampler = RandomSampler(
+        num=10, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=True)
+
+    sample_result = sampler.sample(assign_result, bboxes, gt_bboxes, gt_labels)
+
+    assert len(sample_result.pos_bboxes) == len(sample_result.pos_inds)
+    assert len(sample_result.neg_bboxes) == len(sample_result.neg_inds)
+
+
+def test_random_sampler_empty_gt():
+    assigner = MaxIoUAssigner(
+        pos_iou_thr=0.5,
+        neg_iou_thr=0.5,
+        ignore_iof_thr=0.5,
+        ignore_wrt_candidates=False,
+    )
+    bboxes = torch.FloatTensor([
+        [0, 0, 10, 10],
+        [10, 10, 20, 20],
+        [5, 5, 15, 15],
+        [32, 32, 38, 42],
+    ])
+    gt_bboxes = torch.empty(0, 4)
+    gt_labels = torch.empty(0, ).long()
+    assign_result = assigner.assign(bboxes, gt_bboxes, gt_labels=gt_labels)
+
+    sampler = RandomSampler(
+        num=10, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=True)
+
+    sample_result = sampler.sample(assign_result, bboxes, gt_bboxes, gt_labels)
+
+    assert len(sample_result.pos_bboxes) == len(sample_result.pos_inds)
+    assert len(sample_result.neg_bboxes) == len(sample_result.neg_inds)
+
+
+def test_random_sampler_empty_pred():
+    assigner = MaxIoUAssigner(
+        pos_iou_thr=0.5,
+        neg_iou_thr=0.5,
+        ignore_iof_thr=0.5,
+        ignore_wrt_candidates=False,
+    )
+    bboxes = torch.empty(0, 4)
+    gt_bboxes = torch.FloatTensor([
+        [0, 0, 10, 9],
+        [0, 10, 10, 19],
+    ])
+    gt_labels = torch.LongTensor([1, 2])
+    assign_result = assigner.assign(bboxes, gt_bboxes, gt_labels=gt_labels)
+
+    sampler = RandomSampler(
+        num=10, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=True)
+
+    sample_result = sampler.sample(assign_result, bboxes, gt_bboxes, gt_labels)
+
+    assert len(sample_result.pos_bboxes) == len(sample_result.pos_inds)
+    assert len(sample_result.neg_bboxes) == len(sample_result.neg_inds)
+
+
+def _context_for_ohem():
+    try:
+        from test_forward import _get_detector_cfg
+    except ImportError:
+        # Hack: grab testing utils from test_forward to make a context for ohem
+        import sys
+        from os.path import dirname
+        sys.path.insert(0, dirname(__file__))
+        from test_forward import _get_detector_cfg
+    model, train_cfg, test_cfg = _get_detector_cfg(
+        'faster_rcnn_ohem_r50_fpn_1x.py')
+    model['pretrained'] = None
+    # torchvision roi align supports CPU
+    model['bbox_roi_extractor']['roi_layer']['use_torchvision'] = True
+    from mmdet.models import build_detector
+    context = build_detector(model, train_cfg=train_cfg, test_cfg=test_cfg)
+    return context
+
+
+def test_ohem_sampler():
+
+    assigner = MaxIoUAssigner(
+        pos_iou_thr=0.5,
+        neg_iou_thr=0.5,
+        ignore_iof_thr=0.5,
+        ignore_wrt_candidates=False,
+    )
+    bboxes = torch.FloatTensor([
+        [0, 0, 10, 10],
+        [10, 10, 20, 20],
+        [5, 5, 15, 15],
+        [32, 32, 38, 42],
+    ])
+    gt_bboxes = torch.FloatTensor([
+        [0, 0, 10, 9],
+        [0, 10, 10, 19],
+    ])
+    gt_labels = torch.LongTensor([1, 2])
+    gt_bboxes_ignore = torch.Tensor([
+        [30, 30, 40, 40],
+    ])
+    assign_result = assigner.assign(
+        bboxes,
+        gt_bboxes,
+        gt_bboxes_ignore=gt_bboxes_ignore,
+        gt_labels=gt_labels)
+
+    context = _context_for_ohem()
+
+    sampler = OHEMSampler(
+        num=10,
+        pos_fraction=0.5,
+        context=context,
+        neg_pos_ub=-1,
+        add_gt_as_proposals=True)
+
+    feats = [torch.rand(1, 256, int(2**i), int(2**i)) for i in [6, 5, 4, 3, 2]]
+    sample_result = sampler.sample(
+        assign_result, bboxes, gt_bboxes, gt_labels, feats=feats)
+
+    assert len(sample_result.pos_bboxes) == len(sample_result.pos_inds)
+    assert len(sample_result.neg_bboxes) == len(sample_result.neg_inds)
+
+
+def test_ohem_sampler_empty_gt():
+
+    assigner = MaxIoUAssigner(
+        pos_iou_thr=0.5,
+        neg_iou_thr=0.5,
+        ignore_iof_thr=0.5,
+        ignore_wrt_candidates=False,
+    )
+    bboxes = torch.FloatTensor([
+        [0, 0, 10, 10],
+        [10, 10, 20, 20],
+        [5, 5, 15, 15],
+        [32, 32, 38, 42],
+    ])
+    gt_bboxes = torch.empty(0, 4)
+    gt_labels = torch.LongTensor([])
+    gt_bboxes_ignore = torch.Tensor([])
+    assign_result = assigner.assign(
+        bboxes,
+        gt_bboxes,
+        gt_bboxes_ignore=gt_bboxes_ignore,
+        gt_labels=gt_labels)
+
+    context = _context_for_ohem()
+
+    sampler = OHEMSampler(
+        num=10,
+        pos_fraction=0.5,
+        context=context,
+        neg_pos_ub=-1,
+        add_gt_as_proposals=True)
+
+    feats = [torch.rand(1, 256, int(2**i), int(2**i)) for i in [6, 5, 4, 3, 2]]
+
+    sample_result = sampler.sample(
+        assign_result, bboxes, gt_bboxes, gt_labels, feats=feats)
+
+    assert len(sample_result.pos_bboxes) == len(sample_result.pos_inds)
+    assert len(sample_result.neg_bboxes) == len(sample_result.neg_inds)
+
+
+def test_ohem_sampler_empty_pred():
+    assigner = MaxIoUAssigner(
+        pos_iou_thr=0.5,
+        neg_iou_thr=0.5,
+        ignore_iof_thr=0.5,
+        ignore_wrt_candidates=False,
+    )
+    bboxes = torch.empty(0, 4)
+    gt_bboxes = torch.FloatTensor([
+        [0, 0, 10, 10],
+        [10, 10, 20, 20],
+        [5, 5, 15, 15],
+        [32, 32, 38, 42],
+    ])
+    gt_labels = torch.LongTensor([1, 2, 2, 3])
+    gt_bboxes_ignore = torch.Tensor([])
+    assign_result = assigner.assign(
+        bboxes,
+        gt_bboxes,
+        gt_bboxes_ignore=gt_bboxes_ignore,
+        gt_labels=gt_labels)
+
+    context = _context_for_ohem()
+
+    sampler = OHEMSampler(
+        num=10,
+        pos_fraction=0.5,
+        context=context,
+        neg_pos_ub=-1,
+        add_gt_as_proposals=True)
+
+    feats = [torch.rand(1, 256, int(2**i), int(2**i)) for i in [6, 5, 4, 3, 2]]
+
+    sample_result = sampler.sample(
+        assign_result, bboxes, gt_bboxes, gt_labels, feats=feats)
+
+    assert len(sample_result.pos_bboxes) == len(sample_result.pos_inds)
+    assert len(sample_result.neg_bboxes) == len(sample_result.neg_inds)
+
+
+def test_random_sample_result():
+    from mmdet.core.bbox.samplers.sampling_result import SamplingResult
+    SamplingResult.random(num_gts=0, num_preds=0)
+    SamplingResult.random(num_gts=0, num_preds=3)
+    SamplingResult.random(num_gts=3, num_preds=3)
+    SamplingResult.random(num_gts=0, num_preds=3)
+    SamplingResult.random(num_gts=7, num_preds=7)
+    SamplingResult.random(num_gts=7, num_preds=64)
+    SamplingResult.random(num_gts=24, num_preds=3)
+
+    for i in range(3):
+        SamplingResult.random(rng=i)

tests/test_utils.py

+import numpy.testing as npt
+
+from mmdet.utils.flops_counter import params_to_string
+
+
+def test_params_to_string():
+    npt.assert_equal(params_to_string(1e9), '1000.0 M')
+    npt.assert_equal(params_to_string(2e5), '200.0 k')
+    npt.assert_equal(params_to_string(3e-9), '3e-09')

tools/analyze_logs.py

@@ -32,28 +32,27 @@ def cal_train_time(log_dicts, args):
 
 
 def plot_curve(log_dicts, args):
-    if args.backend is not None:
-        plt.switch_backend(args.backend)
-    sns.set_style(args.style)
+    if args['backend'] is not None:
+        plt.switch_backend(args['backend'])
     # if legend is None, use {filename}_{key} as legend
-    legend = args.legend
+    legend = args['legend']
     if legend is None:
         legend = []
-        for json_log in args.json_logs:
-            for metric in args.keys:
+        for json_log in args['json_logs']:
+            for metric in args['keys']:
                 legend.append('{}_{}'.format(json_log, metric))
-    assert len(legend) == (len(args.json_logs) * len(args.keys))
-    metrics = args.keys
+    assert len(legend) == (len(args['json_logs']) * len(args['keys']))
+    metrics = args['keys']
 
     num_metrics = len(metrics)
     for i, log_dict in enumerate(log_dicts):
         epochs = list(log_dict.keys())
         for j, metric in enumerate(metrics):
             print('plot curve of {}, metric is {}'.format(
-                args.json_logs[i], metric))
+                args['json_logs'][i], metric))
             if metric not in log_dict[epochs[0]]:
                 raise KeyError('{} does not contain metric {}'.format(
-                    args.json_logs[i], metric))
+                    args['json_logs'][i], metric))
 
             if 'mAP' in metric:
                 xs = np.arange(1, max(epochs) + 1)
@@ -81,13 +80,13 @@ def plot_curve(log_dicts, args):
                 plt.plot(
                     xs, ys, label=legend[i * num_metrics + j], linewidth=0.5)
             plt.legend()
-        if args.title is not None:
-            plt.title(args.title)
-    if args.out is None:
+        if args['title'] is not None:
+            plt.title(args['title'])
+    if args['out'] is None:
         plt.show()
     else:
-        print('save curve to: {}'.format(args.out))
-        plt.savefig(args.out)
+        print('save curve to: {}'.format(args['out']))
+        plt.savefig(args['out'])
         plt.cla()
 
 

tools/coco_error_analysis.py

+import copy
+import os
+from argparse import ArgumentParser
+from multiprocessing import Pool
+
+import matplotlib.pyplot as plt
+import numpy as np
+from pycocotools.coco import COCO
+from pycocotools.cocoeval import COCOeval
+
+
+def makeplot(rs, ps, outDir, class_name, iou_type):
+    cs = np.vstack([
+        np.ones((2, 3)),
+        np.array([.31, .51, .74]),
+        np.array([.75, .31, .30]),
+        np.array([.36, .90, .38]),
+        np.array([.50, .39, .64]),
+        np.array([1, .6, 0])
+    ])
+    areaNames = ['allarea', 'small', 'medium', 'large']
+    types = ['C75', 'C50', 'Loc', 'Sim', 'Oth', 'BG', 'FN']
+    for i in range(len(areaNames)):
+        area_ps = ps[..., i, 0]
+        figure_tile = iou_type + '-' + class_name + '-' + areaNames[i]
+        aps = [ps_.mean() for ps_ in area_ps]
+        ps_curve = [
+            ps_.mean(axis=1) if ps_.ndim > 1 else ps_ for ps_ in area_ps
+        ]
+        ps_curve.insert(0, np.zeros(ps_curve[0].shape))
+        fig = plt.figure()
+        ax = plt.subplot(111)
+        for k in range(len(types)):
+            ax.plot(rs, ps_curve[k + 1], color=[0, 0, 0], linewidth=0.5)
+            ax.fill_between(
+                rs,
+                ps_curve[k],
+                ps_curve[k + 1],
+                color=cs[k],
+                label=str('[{:.3f}'.format(aps[k]) + ']' + types[k]))
+        plt.xlabel('recall')
+        plt.ylabel('precision')
+        plt.xlim(0, 1.)
+        plt.ylim(0, 1.)
+        plt.title(figure_tile)
+        plt.legend()
+        # plt.show()
+        fig.savefig(outDir + '/{}.png'.format(figure_tile))
+        plt.close(fig)
+
+
+def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
+    nm = cocoGt.loadCats(catId)[0]
+    print('--------------analyzing {}-{}---------------'.format(
+        k + 1, nm['name']))
+    ps_ = {}
+    dt = copy.deepcopy(cocoDt)
+    nm = cocoGt.loadCats(catId)[0]
+    imgIds = cocoGt.getImgIds()
+    dt_anns = dt.dataset['annotations']
+    select_dt_anns = []
+    for ann in dt_anns:
+        if ann['category_id'] == catId:
+            select_dt_anns.append(ann)
+    dt.dataset['annotations'] = select_dt_anns
+    dt.createIndex()
+    # compute precision but ignore superclass confusion
+    gt = copy.deepcopy(cocoGt)
+    child_catIds = gt.getCatIds(supNms=[nm['supercategory']])
+    for idx, ann in enumerate(gt.dataset['annotations']):
+        if (ann['category_id'] in child_catIds
+                and ann['category_id'] != catId):
+            gt.dataset['annotations'][idx]['ignore'] = 1
+            gt.dataset['annotations'][idx]['iscrowd'] = 1
+            gt.dataset['annotations'][idx]['category_id'] = catId
+    cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type)
+    cocoEval.params.imgIds = imgIds
+    cocoEval.params.maxDets = [100]
+    cocoEval.params.iouThrs = [.1]
+    cocoEval.params.useCats = 1
+    cocoEval.evaluate()
+    cocoEval.accumulate()
+    ps_supercategory = cocoEval.eval['precision'][0, :, k, :, :]
+    ps_['ps_supercategory'] = ps_supercategory
+    # compute precision but ignore any class confusion
+    gt = copy.deepcopy(cocoGt)
+    for idx, ann in enumerate(gt.dataset['annotations']):
+        if ann['category_id'] != catId:
+            gt.dataset['annotations'][idx]['ignore'] = 1
+            gt.dataset['annotations'][idx]['iscrowd'] = 1
+            gt.dataset['annotations'][idx]['category_id'] = catId
+    cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type)
+    cocoEval.params.imgIds = imgIds
+    cocoEval.params.maxDets = [100]
+    cocoEval.params.iouThrs = [.1]
+    cocoEval.params.useCats = 1
+    cocoEval.evaluate()
+    cocoEval.accumulate()
+    ps_allcategory = cocoEval.eval['precision'][0, :, k, :, :]
+    ps_['ps_allcategory'] = ps_allcategory
+    return k, ps_
+
+
+def analyze_results(res_file, ann_file, res_types, out_dir):
+    for res_type in res_types:
+        assert res_type in ['bbox', 'segm']
+
+    directory = os.path.dirname(out_dir + '/')
+    if not os.path.exists(directory):
+        print('-------------create {}-----------------'.format(out_dir))
+        os.makedirs(directory)
+
+    cocoGt = COCO(ann_file)
+    cocoDt = cocoGt.loadRes(res_file)
+    imgIds = cocoGt.getImgIds()
+    for res_type in res_types:
+        res_out_dir = out_dir + '/' + res_type + '/'
+        res_directory = os.path.dirname(res_out_dir)
+        if not os.path.exists(res_directory):
+            print(
+                '-------------create {}-----------------'.format(res_out_dir))
+            os.makedirs(res_directory)
+        iou_type = res_type
+        cocoEval = COCOeval(
+            copy.deepcopy(cocoGt), copy.deepcopy(cocoDt), iou_type)
+        cocoEval.params.imgIds = imgIds
+        cocoEval.params.iouThrs = [.75, .5, .1]
+        cocoEval.params.maxDets = [100]
+        cocoEval.evaluate()
+        cocoEval.accumulate()
+        ps = cocoEval.eval['precision']
+        ps = np.vstack([ps, np.zeros((4, *ps.shape[1:]))])
+        catIds = cocoGt.getCatIds()
+        recThrs = cocoEval.params.recThrs
+        with Pool(processes=48) as pool:
+            args = [(k, cocoDt, cocoGt, catId, iou_type)
+                    for k, catId in enumerate(catIds)]
+            analyze_results = pool.starmap(analyze_individual_category, args)
+        for k, catId in enumerate(catIds):
+            nm = cocoGt.loadCats(catId)[0]
+            print('--------------saving {}-{}---------------'.format(
+                k + 1, nm['name']))
+            analyze_result = analyze_results[k]
+            assert k == analyze_result[0]
+            ps_supercategory = analyze_result[1]['ps_supercategory']
+            ps_allcategory = analyze_result[1]['ps_allcategory']
+            # compute precision but ignore superclass confusion
+            ps[3, :, k, :, :] = ps_supercategory
+            # compute precision but ignore any class confusion
+            ps[4, :, k, :, :] = ps_allcategory
+            # fill in background and false negative errors and plot
+            ps[ps == -1] = 0
+            ps[5, :, k, :, :] = (ps[4, :, k, :, :] > 0)
+            ps[6, :, k, :, :] = 1.0
+            makeplot(recThrs, ps[:, :, k], res_out_dir, nm['name'], iou_type)
+        makeplot(recThrs, ps, res_out_dir, 'allclass', iou_type)
+
+
+def main():
+    parser = ArgumentParser(description='COCO Error Analysis Tool')
+    parser.add_argument('result', help='result file (json format) path')
+    parser.add_argument('out_dir', help='dir to save analyze result images')
+    parser.add_argument(
+        '--ann',
+        default='data/coco/annotations/instances_val2017.json',
+        help='annotation file path')
+    parser.add_argument(
+        '--types', type=str, nargs='+', default=['bbox'], help='result types')
+    args = parser.parse_args()
+    analyze_results(args.result, args.ann, args.types, out_dir=args.out_dir)
+
+
+if __name__ == '__main__':
+    main()

tools/coco_eval.py

@@ -20,8 +20,10 @@ def main():
         nargs='+',
         default=[100, 300, 1000],
         help='proposal numbers, only used for recall evaluation')
+    parser.add_argument(
+        '--classwise', action='store_true', help='whether eval class wise ap')
     args = parser.parse_args()
-    coco_eval(args.result, args.types, args.ann, args.max_dets)
+    coco_eval(args.result, args.types, args.ann, args.max_dets, args.classwise)
 
 
 if __name__ == '__main__':

tools/collect_env.py

+import os.path as osp
+import subprocess
+import sys
+from collections import defaultdict
+
+import cv2
+import mmcv
+import torch
+import torchvision
+
+import mmdet
+from mmdet.ops import get_compiler_version, get_compiling_cuda_version
+
+
+def collect_env():
+    env_info = {}
+    env_info['sys.platform'] = sys.platform
+    env_info['Python'] = sys.version.replace('\n', '')
+
+    cuda_available = torch.cuda.is_available()
+    env_info['CUDA available'] = cuda_available
+
+    if cuda_available:
+        from torch.utils.cpp_extension import CUDA_HOME
+        env_info['CUDA_HOME'] = CUDA_HOME
+
+        if CUDA_HOME is not None and osp.isdir(CUDA_HOME):
+            try:
+                nvcc = osp.join(CUDA_HOME, 'bin/nvcc')
+                nvcc = subprocess.check_output(
+                    '"{}" -V | tail -n1'.format(nvcc), shell=True)
+                nvcc = nvcc.decode('utf-8').strip()
+            except subprocess.SubprocessError:
+                nvcc = 'Not Available'
+            env_info['NVCC'] = nvcc
+
+        devices = defaultdict(list)
+        for k in range(torch.cuda.device_count()):
+            devices[torch.cuda.get_device_name(k)].append(str(k))
+        for name, devids in devices.items():
+            env_info['GPU ' + ','.join(devids)] = name
+
+    gcc = subprocess.check_output('gcc --version | head -n1', shell=True)
+    gcc = gcc.decode('utf-8').strip()
+    env_info['GCC'] = gcc
+
+    env_info['PyTorch'] = torch.__version__
+    env_info['PyTorch compiling details'] = torch.__config__.show()
+
+    env_info['TorchVision'] = torchvision.__version__
+
+    env_info['OpenCV'] = cv2.__version__
+
+    env_info['MMCV'] = mmcv.__version__
+    env_info['MMDetection'] = mmdet.__version__
+    env_info['MMDetection Compiler'] = get_compiler_version()
+    env_info['MMDetection CUDA Compiler'] = get_compiling_cuda_version()
+
+    for name, val in env_info.items():
+        print('{}: {}'.format(name, val))
+
+
+if __name__ == "__main__":
+    collect_env()

tools/dist_test.sh

@@ -5,6 +5,7 @@ PYTHON=${PYTHON:-"python"}
 CONFIG=$1
 CHECKPOINT=$2
 GPUS=$3
+PORT=${PORT:-29500}
 
-$PYTHON -m torch.distributed.launch --nproc_per_node=$GPUS \
+$PYTHON -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \
     $(dirname "$0")/test.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4}

tools/dist_train.sh

 #!/usr/bin/env bash
 
-PYTHON=${PYTHON:-"python"}
+PYTHON=${PYTHON:-"python -W ignore"}
 
 CONFIG=$1
 GPUS=$2
+PORT=${PORT:-29500}
 
-$PYTHON -m torch.distributed.launch --nproc_per_node=$GPUS \
+$PYTHON -W ignore -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \
     $(dirname "$0")/train.py $CONFIG --launcher pytorch ${@:3}

tools/get_flops.py

+import argparse
+
+from mmcv import Config
+
+from mmdet.models import build_detector
+from mmdet.utils import get_model_complexity_info
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Train a detector')
+    parser.add_argument('config', help='train config file path')
+    parser.add_argument(
+        '--shape',
+        type=int,
+        nargs='+',
+        default=[1280, 800],
+        help='input image size')
+    args = parser.parse_args()
+    return args
+
+
+def main():
+
+    args = parse_args()
+
+    if len(args.shape) == 1:
+        input_shape = (3, args.shape[0], args.shape[0])
+    elif len(args.shape) == 2:
+        input_shape = (3, ) + tuple(args.shape)
+    else:
+        raise ValueError('invalid input shape')
+
+    cfg = Config.fromfile(args.config)
+    model = build_detector(
+        cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg).cuda()
+    model.eval()
+
+    if hasattr(model, 'forward_dummy'):
+        model.forward = model.forward_dummy
+    else:
+        raise NotImplementedError(
+            'FLOPs counter is currently not currently supported with {}'.
+            format(model.__class__.__name__))
+
+    flops, params = get_model_complexity_info(model, input_shape)
+    split_line = '=' * 30
+    print('{0}\nInput shape: {1}\nFlops: {2}\nParams: {3}\n{0}'.format(
+        split_line, input_shape, flops, params))
+    print('!!!Please be cautious if you use the results in papers. '
+          'You may need to check if all ops are supported and verify that the '
+          'flops computation is correct.')
+
+
+if __name__ == '__main__':
+    main()

tools/test.py → tools/infer.py

 import argparse
 import os
 import os.path as osp
+import pickle
 import shutil
 import tempfile
 
@@ -8,12 +9,30 @@ import mmcv
 import torch
 import torch.distributed as dist
 from mmcv.parallel import MMDataParallel, MMDistributedDataParallel
-from mmcv.runner import get_dist_info, load_checkpoint
+from mmcv.runner import get_dist_info, init_dist, load_checkpoint
 
-from mmdet.apis import init_dist
 from mmdet.core import coco_eval, results2json, wrap_fp16_model
 from mmdet.datasets import build_dataloader, build_dataset
 from mmdet.models import build_detector
+import glob
+import json
+test_images_path = os.getenv("AICROWD_TEST_IMAGES_PATH", False)
+predictions_output_path = os.getenv("AICROWD_PREDICTIONS_OUTPUT_PATH", False)
+print(predictions_output_path)
+annotations = {'categories': [], 'info': {}, 'images': []}
+for item in glob.glob(test_images_path+'/*.jpg'):
+    image_dict = dict()
+
+    img = mmcv.imread(item)
+    height,width,__ = img.shape
+    id = int(os.path.basename(item).split('.')[0])
+    image_dict['id'] = id
+    image_dict['file_name'] = os.path.basename(item)
+    image_dict['width'] = width
+    image_dict['height'] = height
+    annotations['images'].append(image_dict)
+annotations['categories'] = json.loads(open("all_categories.json").read())
+json.dump(annotations, open('test_annotations.json', 'w'))
 
 
 def single_gpu_test(model, data_loader, show=False):
@@ -27,7 +46,7 @@ def single_gpu_test(model, data_loader, show=False):
         results.append(result)
 
         if show:
-            model.module.show_result(data, result, dataset.img_norm_cfg)
+            model.module.show_result(data, result)
 
         batch_size = data['img'][0].size(0)
         for _ in range(batch_size):
@@ -35,7 +54,25 @@ def single_gpu_test(model, data_loader, show=False):
     return results
 
 
-def multi_gpu_test(model, data_loader, tmpdir=None):
+def multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False):
+    """Test model with multiple gpus.
+
+    This method tests model with multiple gpus and collects the results
+    under two different modes: gpu and cpu modes. By setting 'gpu_collect=True'
+    it encodes results to gpu tensors and use gpu communication for results
+    collection. On cpu mode it saves the results on different gpus to 'tmpdir'
+    and collects them by the rank 0 worker.
+
+    Args:
+        model (nn.Module): Model to be tested.
+        data_loader (nn.Dataloader): Pytorch data loader.
+        tmpdir (str): Path of directory to save the temporary results from
+            different gpus under cpu mode.
+        gpu_collect (bool): Option to use either gpu or cpu to collect results.
+
+    Returns:
+        list: The prediction results.
+    """
     model.eval()
     results = []
     dataset = data_loader.dataset
@@ -53,12 +90,14 @@ def multi_gpu_test(model, data_loader, tmpdir=None):
                 prog_bar.update()
 
     # collect results from all ranks
-    results = collect_results(results, len(dataset), tmpdir)
-
+    if gpu_collect:
+        results = collect_results_gpu(results, len(dataset))
+    else:
+        results = collect_results_cpu(results, len(dataset), tmpdir)
     return results
 
 
-def collect_results(result_part, size, tmpdir=None):
+def collect_results_cpu(result_part, size, tmpdir=None):
     rank, world_size = get_dist_info()
     # create a tmp dir if it is not specified
     if tmpdir is None:
@@ -100,6 +139,39 @@ def collect_results(result_part, size, tmpdir=None):
         return ordered_results
 
 
+def collect_results_gpu(result_part, size):
+    rank, world_size = get_dist_info()
+    # dump result part to tensor with pickle
+    part_tensor = torch.tensor(
+        bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda')
+    # gather all result part tensor shape
+    shape_tensor = torch.tensor(part_tensor.shape, device='cuda')
+    shape_list = [shape_tensor.clone() for _ in range(world_size)]
+    dist.all_gather(shape_list, shape_tensor)
+    # padding result part tensor to max length
+    shape_max = torch.tensor(shape_list).max()
+    part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda')
+    part_send[:shape_tensor[0]] = part_tensor
+    part_recv_list = [
+        part_tensor.new_zeros(shape_max) for _ in range(world_size)
+    ]
+    # gather all result part
+    dist.all_gather(part_recv_list, part_send)
+
+    if rank == 0:
+        part_list = []
+        for recv, shape in zip(part_recv_list, shape_list):
+            part_list.append(
+                pickle.loads(recv[:shape[0]].cpu().numpy().tobytes()))
+        # sort the results
+        ordered_results = []
+        for res in zip(*part_list):
+            ordered_results.extend(list(res))
+        # the dataloader may pad some samples
+        ordered_results = ordered_results[:size]
+        return ordered_results
+
+
 def parse_args():
     parser = argparse.ArgumentParser(description='MMDet test detector')
     parser.add_argument('config', help='test config file path')
@@ -116,6 +188,10 @@ def parse_args():
         choices=['proposal', 'proposal_fast', 'bbox', 'segm', 'keypoints'],
         help='eval types')
     parser.add_argument('--show', action='store_true', help='show results')
+    parser.add_argument(
+        '--gpu_collect',
+        action='store_true',
+        help='whether to use gpu to collect results')
     parser.add_argument('--tmpdir', help='tmp dir for writing some results')
     parser.add_argument(
         '--launcher',
@@ -148,7 +224,8 @@ def main():
         torch.backends.cudnn.benchmark = True
     cfg.model.pretrained = None
     cfg.data.test.test_mode = True
-
+    cfg.data.test.ann_file = 'test_annotations.json'
+    cfg.data.test.img_prefix = test_images_path
     # init distributed env first, since logger depends on the dist info.
     if args.launcher == 'none':
         distributed = False
@@ -174,17 +251,14 @@ def main():
     checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu')
     # old versions did not save class info in checkpoints, this walkaround is
     # for backward compatibility
-    if 'CLASSES' in checkpoint['meta']:
-        model.CLASSES = checkpoint['meta']['CLASSES']
-    else:
-        model.CLASSES = dataset.CLASSES
-
+    model.CLASSES = [category['name'] for category in annotations['categories']]
     if not distributed:
         model = MMDataParallel(model, device_ids=[0])
         outputs = single_gpu_test(model, data_loader, args.show)
     else:
         model = MMDistributedDataParallel(model.cuda())
-        outputs = multi_gpu_test(model, data_loader, args.tmpdir)
+        outputs = multi_gpu_test(model, data_loader, args.tmpdir,
+                                 args.gpu_collect)
 
     rank, _ = get_dist_info()
     if args.out and rank == 0:
@@ -212,13 +286,13 @@ def main():
     # Save predictions in the COCO json format
     if args.json_out and rank == 0:
         if not isinstance(outputs[0], dict):
-            results2json(dataset, outputs, args.json_out)
+            response = results2json(dataset, outputs, args.json_out)
         else:
             for name in outputs[0]:
                 outputs_ = [out[name] for out in outputs]
                 result_file = args.json_out + '.{}'.format(name)
-                results2json(dataset, outputs_, result_file)
-
+                response = results2json(dataset, outputs_, result_file)
+        shutil.move(response['segm'], predictions_output_path)
 
 if __name__ == '__main__':
     main()

tools/robustness_eval.py

+import os.path as osp
+from argparse import ArgumentParser
+
+import mmcv
+import numpy as np
+
+
+def print_coco_results(results):
+
+    def _print(result, ap=1, iouThr=None, areaRng='all', maxDets=100):
+        iStr = ' {:<18} {} @[ IoU={:<9} | \
+        area={:>6s} | maxDets={:>3d} ] = {:0.3f}'
+
+        titleStr = 'Average Precision' if ap == 1 else 'Average Recall'
+        typeStr = '(AP)' if ap == 1 else '(AR)'
+        iouStr = '{:0.2f}:{:0.2f}'.format(.5, .95) \
+            if iouThr is None else '{:0.2f}'.format(iouThr)
+        print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, result))
+
+    stats = np.zeros((12, ))
+    stats[0] = _print(results[0], 1)
+    stats[1] = _print(results[1], 1, iouThr=.5)
+    stats[2] = _print(results[2], 1, iouThr=.75)
+    stats[3] = _print(results[3], 1, areaRng='small')
+    stats[4] = _print(results[4], 1, areaRng='medium')
+    stats[5] = _print(results[5], 1, areaRng='large')
+    stats[6] = _print(results[6], 0, maxDets=1)
+    stats[7] = _print(results[7], 0, maxDets=10)
+    stats[8] = _print(results[8], 0)
+    stats[9] = _print(results[9], 0, areaRng='small')
+    stats[10] = _print(results[10], 0, areaRng='medium')
+    stats[11] = _print(results[11], 0, areaRng='large')
+
+
+def get_coco_style_results(filename,
+                           task='bbox',
+                           metric=None,
+                           prints='mPC',
+                           aggregate='benchmark'):
+
+    assert aggregate in ['benchmark', 'all']
+
+    if prints == 'all':
+        prints = ['P', 'mPC', 'rPC']
+    elif isinstance(prints, str):
+        prints = [prints]
+    for p in prints:
+        assert p in ['P', 'mPC', 'rPC']
+
+    if metric is None:
+        metrics = [
+            'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', 'AR100',
+            'ARs', 'ARm', 'ARl'
+        ]
+    elif isinstance(metric, list):
+        metrics = metric
+    else:
+        metrics = [metric]
+
+    for metric_name in metrics:
+        assert metric_name in [
+            'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', 'AR100',
+            'ARs', 'ARm', 'ARl'
+        ]
+
+    eval_output = mmcv.load(filename)
+
+    num_distortions = len(list(eval_output.keys()))
+    results = np.zeros((num_distortions, 6, len(metrics)), dtype='float32')
+
+    for corr_i, distortion in enumerate(eval_output):
+        for severity in eval_output[distortion]:
+            for metric_j, metric_name in enumerate(metrics):
+                mAP = eval_output[distortion][severity][task][metric_name]
+                results[corr_i, severity, metric_j] = mAP
+
+    P = results[0, 0, :]
+    if aggregate == 'benchmark':
+        mPC = np.mean(results[:15, 1:, :], axis=(0, 1))
+    else:
+        mPC = np.mean(results[:, 1:, :], axis=(0, 1))
+    rPC = mPC / P
+
+    print('\nmodel: {}'.format(osp.basename(filename)))
+    if metric is None:
+        if 'P' in prints:
+            print('Performance on Clean Data [P] ({})'.format(task))
+            print_coco_results(P)
+        if 'mPC' in prints:
+            print('Mean Performance under Corruption [mPC] ({})'.format(task))
+            print_coco_results(mPC)
+        if 'rPC' in prints:
+            print('Realtive Performance under Corruption [rPC] ({})'.format(
+                task))
+            print_coco_results(rPC)
+    else:
+        if 'P' in prints:
+            print('Performance on Clean Data [P] ({})'.format(task))
+            for metric_i, metric_name in enumerate(metrics):
+                print('{:5} =  {:0.3f}'.format(metric_name, P[metric_i]))
+        if 'mPC' in prints:
+            print('Mean Performance under Corruption [mPC] ({})'.format(task))
+            for metric_i, metric_name in enumerate(metrics):
+                print('{:5} =  {:0.3f}'.format(metric_name, mPC[metric_i]))
+        if 'rPC' in prints:
+            print('Relative Performance under Corruption [rPC] ({})'.format(
+                task))
+            for metric_i, metric_name in enumerate(metrics):
+                print('{:5} => {:0.1f} %'.format(metric_name,
+                                                 rPC[metric_i] * 100))
+
+    return results
+
+
+def get_voc_style_results(filename, prints='mPC', aggregate='benchmark'):
+
+    assert aggregate in ['benchmark', 'all']
+
+    if prints == 'all':
+        prints = ['P', 'mPC', 'rPC']
+    elif isinstance(prints, str):
+        prints = [prints]
+    for p in prints:
+        assert p in ['P', 'mPC', 'rPC']
+
+    eval_output = mmcv.load(filename)
+
+    num_distortions = len(list(eval_output.keys()))
+    results = np.zeros((num_distortions, 6, 20), dtype='float32')
+
+    for i, distortion in enumerate(eval_output):
+        for severity in eval_output[distortion]:
+            mAP = [
+                eval_output[distortion][severity][j]['ap']
+                for j in range(len(eval_output[distortion][severity]))
+            ]
+            results[i, severity, :] = mAP
+
+    P = results[0, 0, :]
+    if aggregate == 'benchmark':
+        mPC = np.mean(results[:15, 1:, :], axis=(0, 1))
+    else:
+        mPC = np.mean(results[:, 1:, :], axis=(0, 1))
+    rPC = mPC / P
+
+    print('\nmodel: {}'.format(osp.basename(filename)))
+    if 'P' in prints:
+        print('{:48} = {:0.3f}'.format('Performance on Clean Data [P] in AP50',
+                                       np.mean(P)))
+    if 'mPC' in prints:
+        print('{:48} = {:0.3f}'.format(
+            'Mean Performance under Corruption [mPC] in AP50', np.mean(mPC)))
+    if 'rPC' in prints:
+        print('{:48} = {:0.1f}'.format(
+            'Realtive Performance under Corruption [rPC] in %',
+            np.mean(rPC) * 100))
+
+    return np.mean(results, axis=2, keepdims=True)
+
+
+def get_results(filename,
+                dataset='coco',
+                task='bbox',
+                metric=None,
+                prints='mPC',
+                aggregate='benchmark'):
+    assert dataset in ['coco', 'voc', 'cityscapes']
+
+    if dataset in ['coco', 'cityscapes']:
+        results = get_coco_style_results(
+            filename,
+            task=task,
+            metric=metric,
+            prints=prints,
+            aggregate=aggregate)
+    elif dataset == 'voc':
+        if task != 'bbox':
+            print('Only bbox analysis is supported for Pascal VOC')
+            print('Will report bbox results\n')
+        if metric not in [None, ['AP'], ['AP50']]:
+            print('Only the AP50 metric is supported for Pascal VOC')
+            print('Will report AP50 metric\n')
+        results = get_voc_style_results(
+            filename, prints=prints, aggregate=aggregate)
+
+    return results
+
+
+def get_distortions_from_file(filename):
+
+    eval_output = mmcv.load(filename)
+
+    return get_distortions_from_results(eval_output)
+
+
+def get_distortions_from_results(eval_output):
+    distortions = []
+    for i, distortion in enumerate(eval_output):
+        distortions.append(distortion.replace("_", " "))
+    return distortions
+
+
+def main():
+    parser = ArgumentParser(description='Corruption Result Analysis')
+    parser.add_argument('filename', help='result file path')
+    parser.add_argument(
+        '--dataset',
+        type=str,
+        choices=['coco', 'voc', 'cityscapes'],
+        default='coco',
+        help='dataset type')
+    parser.add_argument(
+        '--task',
+        type=str,
+        nargs='+',
+        choices=['bbox', 'segm'],
+        default=['bbox'],
+        help='task to report')
+    parser.add_argument(
+        '--metric',
+        nargs='+',
+        choices=[
+            None, 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10',
+            'AR100', 'ARs', 'ARm', 'ARl'
+        ],
+        default=None,
+        help='metric to report')
+    parser.add_argument(
+        '--prints',
+        type=str,
+        nargs='+',
+        choices=['P', 'mPC', 'rPC'],
+        default='mPC',
+        help='corruption benchmark metric to print')
+    parser.add_argument(
+        '--aggregate',
+        type=str,
+        choices=['all', 'benchmark'],
+        default='benchmark',
+        help='aggregate all results or only those \
+        for benchmark corruptions')
+
+    args = parser.parse_args()
+
+    for task in args.task:
+        get_results(
+            args.filename,
+            dataset=args.dataset,
+            task=task,
+            metric=args.metric,
+            prints=args.prints,
+            aggregate=args.aggregate)
+
+
+if __name__ == '__main__':
+    main()

tools/test_robustness.py

+import argparse
+import copy
+import os
+import os.path as osp
+import shutil
+import tempfile
+
+import mmcv
+import numpy as np
+import torch
+import torch.distributed as dist
+from mmcv.parallel import MMDataParallel, MMDistributedDataParallel
+from mmcv.runner import get_dist_info, init_dist, load_checkpoint
+from pycocotools.coco import COCO
+from pycocotools.cocoeval import COCOeval
+from robustness_eval import get_results
+
+from mmdet import datasets
+from mmdet.apis import set_random_seed
+from mmdet.core import (eval_map, fast_eval_recall, results2json,
+                        wrap_fp16_model)
+from mmdet.datasets import build_dataloader, build_dataset
+from mmdet.models import build_detector
+
+
+def coco_eval_with_return(result_files,
+                          result_types,
+                          coco,
+                          max_dets=(100, 300, 1000)):
+    for res_type in result_types:
+        assert res_type in [
+            'proposal', 'proposal_fast', 'bbox', 'segm', 'keypoints'
+        ]
+
+    if mmcv.is_str(coco):
+        coco = COCO(coco)
+    assert isinstance(coco, COCO)
+
+    if result_types == ['proposal_fast']:
+        ar = fast_eval_recall(result_files, coco, np.array(max_dets))
+        for i, num in enumerate(max_dets):
+            print('AR@{}\t= {:.4f}'.format(num, ar[i]))
+        return
+
+    eval_results = {}
+    for res_type in result_types:
+        result_file = result_files[res_type]
+        assert result_file.endswith('.json')
+
+        coco_dets = coco.loadRes(result_file)
+        img_ids = coco.getImgIds()
+        iou_type = 'bbox' if res_type == 'proposal' else res_type
+        cocoEval = COCOeval(coco, coco_dets, iou_type)
+        cocoEval.params.imgIds = img_ids
+        if res_type == 'proposal':
+            cocoEval.params.useCats = 0
+            cocoEval.params.maxDets = list(max_dets)
+        cocoEval.evaluate()
+        cocoEval.accumulate()
+        cocoEval.summarize()
+        if res_type == 'segm' or res_type == 'bbox':
+            metric_names = [
+                'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10',
+                'AR100', 'ARs', 'ARm', 'ARl'
+            ]
+            eval_results[res_type] = {
+                metric_names[i]: cocoEval.stats[i]
+                for i in range(len(metric_names))
+            }
+        else:
+            eval_results[res_type] = cocoEval.stats
+
+    return eval_results
+
+
+def voc_eval_with_return(result_file,
+                         dataset,
+                         iou_thr=0.5,
+                         logger='print',
+                         only_ap=True):
+    det_results = mmcv.load(result_file)
+    annotations = [dataset.get_ann_info(i) for i in range(len(dataset))]
+    if hasattr(dataset, 'year') and dataset.year == 2007:
+        dataset_name = 'voc07'
+    else:
+        dataset_name = dataset.CLASSES
+    mean_ap, eval_results = eval_map(
+        det_results,
+        annotations,
+        scale_ranges=None,
+        iou_thr=iou_thr,
+        dataset=dataset_name,
+        logger=logger)
+
+    if only_ap:
+        eval_results = [{
+            'ap': eval_results[i]['ap']
+        } for i in range(len(eval_results))]
+
+    return mean_ap, eval_results
+
+
+def single_gpu_test(model, data_loader, show=False):
+    model.eval()
+    results = []
+    dataset = data_loader.dataset
+    prog_bar = mmcv.ProgressBar(len(dataset))
+    for i, data in enumerate(data_loader):
+        with torch.no_grad():
+            result = model(return_loss=False, rescale=not show, **data)
+        results.append(result)
+
+        if show:
+            model.module.show_result(data, result, dataset.img_norm_cfg)
+
+        batch_size = data['img'][0].size(0)
+        for _ in range(batch_size):
+            prog_bar.update()
+    return results
+
+
+def multi_gpu_test(model, data_loader, tmpdir=None):
+    model.eval()
+    results = []
+    dataset = data_loader.dataset
+    rank, world_size = get_dist_info()
+    if rank == 0:
+        prog_bar = mmcv.ProgressBar(len(dataset))
+    for i, data in enumerate(data_loader):
+        with torch.no_grad():
+            result = model(return_loss=False, rescale=True, **data)
+        results.append(result)
+
+        if rank == 0:
+            batch_size = data['img'][0].size(0)
+            for _ in range(batch_size * world_size):
+                prog_bar.update()
+
+    # collect results from all ranks
+    results = collect_results(results, len(dataset), tmpdir)
+
+    return results
+
+
+def collect_results(result_part, size, tmpdir=None):
+    rank, world_size = get_dist_info()
+    # create a tmp dir if it is not specified
+    if tmpdir is None:
+        MAX_LEN = 512
+        # 32 is whitespace
+        dir_tensor = torch.full((MAX_LEN, ),
+                                32,
+                                dtype=torch.uint8,
+                                device='cuda')
+        if rank == 0:
+            tmpdir = tempfile.mkdtemp()
+            tmpdir = torch.tensor(
+                bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda')
+            dir_tensor[:len(tmpdir)] = tmpdir
+        dist.broadcast(dir_tensor, 0)
+        tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip()
+    else:
+        mmcv.mkdir_or_exist(tmpdir)
+    # dump the part result to the dir
+    mmcv.dump(result_part, osp.join(tmpdir, 'part_{}.pkl'.format(rank)))
+    dist.barrier()
+    # collect all parts
+    if rank != 0:
+        return None
+    else:
+        # load results of all parts from tmp dir
+        part_list = []
+        for i in range(world_size):
+            part_file = osp.join(tmpdir, 'part_{}.pkl'.format(i))
+            part_list.append(mmcv.load(part_file))
+        # sort the results
+        ordered_results = []
+        for res in zip(*part_list):
+            ordered_results.extend(list(res))
+        # the dataloader may pad some samples
+        ordered_results = ordered_results[:size]
+        # remove tmp dir
+        shutil.rmtree(tmpdir)
+        return ordered_results
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='MMDet test detector')
+    parser.add_argument('config', help='test config file path')
+    parser.add_argument('checkpoint', help='checkpoint file')
+    parser.add_argument('--out', help='output result file')
+    parser.add_argument(
+        '--corruptions',
+        type=str,
+        nargs='+',
+        default='benchmark',
+        choices=[
+            'all', 'benchmark', 'noise', 'blur', 'weather', 'digital',
+            'holdout', 'None', 'gaussian_noise', 'shot_noise', 'impulse_noise',
+            'defocus_blur', 'glass_blur', 'motion_blur', 'zoom_blur', 'snow',
+            'frost', 'fog', 'brightness', 'contrast', 'elastic_transform',
+            'pixelate', 'jpeg_compression', 'speckle_noise', 'gaussian_blur',
+            'spatter', 'saturate'
+        ],
+        help='corruptions')
+    parser.add_argument(
+        '--severities',
+        type=int,
+        nargs='+',
+        default=[0, 1, 2, 3, 4, 5],
+        help='corruption severity levels')
+    parser.add_argument(
+        '--eval',
+        type=str,
+        nargs='+',
+        choices=['proposal', 'proposal_fast', 'bbox', 'segm', 'keypoints'],
+        help='eval types')
+    parser.add_argument(
+        '--iou-thr',
+        type=float,
+        default=0.5,
+        help='IoU threshold for pascal voc evaluation')
+    parser.add_argument(
+        '--summaries',
+        type=bool,
+        default=False,
+        help='Print summaries for every corruption and severity')
+    parser.add_argument(
+        '--workers', type=int, default=32, help='workers per gpu')
+    parser.add_argument('--show', action='store_true', help='show results')
+    parser.add_argument('--tmpdir', help='tmp dir for writing some results')
+    parser.add_argument('--seed', type=int, default=None, help='random seed')
+    parser.add_argument(
+        '--launcher',
+        choices=['none', 'pytorch', 'slurm', 'mpi'],
+        default='none',
+        help='job launcher')
+    parser.add_argument('--local_rank', type=int, default=0)
+    parser.add_argument(
+        '--final-prints',
+        type=str,
+        nargs='+',
+        choices=['P', 'mPC', 'rPC'],
+        default='mPC',
+        help='corruption benchmark metric to print at the end')
+    parser.add_argument(
+        '--final-prints-aggregate',
+        type=str,
+        choices=['all', 'benchmark'],
+        default='benchmark',
+        help='aggregate all results or only those for benchmark corruptions')
+    args = parser.parse_args()
+    if 'LOCAL_RANK' not in os.environ:
+        os.environ['LOCAL_RANK'] = str(args.local_rank)
+    return args
+
+
+def main():
+    args = parse_args()
+
+    assert args.out or args.show, \
+        ('Please specify at least one operation (save or show the results) '
+         'with the argument "--out" or "--show"')
+
+    if args.out is not None and not args.out.endswith(('.pkl', '.pickle')):
+        raise ValueError('The output file must be a pkl file.')
+
+    cfg = mmcv.Config.fromfile(args.config)
+    # set cudnn_benchmark
+    if cfg.get('cudnn_benchmark', False):
+        torch.backends.cudnn.benchmark = True
+    cfg.model.pretrained = None
+    cfg.data.test.test_mode = True
+    if args.workers == 0:
+        args.workers = cfg.data.workers_per_gpu
+
+    # init distributed env first, since logger depends on the dist info.
+    if args.launcher == 'none':
+        distributed = False
+    else:
+        distributed = True
+        init_dist(args.launcher, **cfg.dist_params)
+
+    # set random seeds
+    if args.seed is not None:
+        set_random_seed(args.seed)
+
+    if 'all' in args.corruptions:
+        corruptions = [
+            'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur',
+            'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog',
+            'brightness', 'contrast', 'elastic_transform', 'pixelate',
+            'jpeg_compression', 'speckle_noise', 'gaussian_blur', 'spatter',
+            'saturate'
+        ]
+    elif 'benchmark' in args.corruptions:
+        corruptions = [
+            'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur',
+            'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog',
+            'brightness', 'contrast', 'elastic_transform', 'pixelate',
+            'jpeg_compression'
+        ]
+    elif 'noise' in args.corruptions:
+        corruptions = ['gaussian_noise', 'shot_noise', 'impulse_noise']
+    elif 'blur' in args.corruptions:
+        corruptions = [
+            'defocus_blur', 'glass_blur', 'motion_blur', 'zoom_blur'
+        ]
+    elif 'weather' in args.corruptions:
+        corruptions = ['snow', 'frost', 'fog', 'brightness']
+    elif 'digital' in args.corruptions:
+        corruptions = [
+            'contrast', 'elastic_transform', 'pixelate', 'jpeg_compression'
+        ]
+    elif 'holdout' in args.corruptions:
+        corruptions = ['speckle_noise', 'gaussian_blur', 'spatter', 'saturate']
+    elif 'None' in args.corruptions:
+        corruptions = ['None']
+        args.severities = [0]
+    else:
+        corruptions = args.corruptions
+
+    aggregated_results = {}
+    for corr_i, corruption in enumerate(corruptions):
+        aggregated_results[corruption] = {}
+        for sev_i, corruption_severity in enumerate(args.severities):
+            # evaluate severity 0 (= no corruption) only once
+            if corr_i > 0 and corruption_severity == 0:
+                aggregated_results[corruption][0] = \
+                    aggregated_results[corruptions[0]][0]
+                continue
+
+            test_data_cfg = copy.deepcopy(cfg.data.test)
+            # assign corruption and severity
+            if corruption_severity > 0:
+                corruption_trans = dict(
+                    type='Corrupt',
+                    corruption=corruption,
+                    severity=corruption_severity)
+                # TODO: hard coded "1", we assume that the first step is
+                # loading images, which needs to be fixed in the future
+                test_data_cfg['pipeline'].insert(1, corruption_trans)
+
+            # print info
+            print('\nTesting {} at severity {}'.format(corruption,
+                                                       corruption_severity))
+
+            # build the dataloader
+            # TODO: support multiple images per gpu
+            #       (only minor changes are needed)
+            dataset = build_dataset(test_data_cfg)
+            data_loader = build_dataloader(
+                dataset,
+                imgs_per_gpu=1,
+                workers_per_gpu=args.workers,
+                dist=distributed,
+                shuffle=False)
+
+            # build the model and load checkpoint
+            model = build_detector(
+                cfg.model, train_cfg=None, test_cfg=cfg.test_cfg)
+            fp16_cfg = cfg.get('fp16', None)
+            if fp16_cfg is not None:
+                wrap_fp16_model(model)
+            checkpoint = load_checkpoint(
+                model, args.checkpoint, map_location='cpu')
+            # old versions did not save class info in checkpoints,
+            # this walkaround is for backward compatibility
+            if 'CLASSES' in checkpoint['meta']:
+                model.CLASSES = checkpoint['meta']['CLASSES']
+            else:
+                model.CLASSES = dataset.CLASSES
+
+            if not distributed:
+                model = MMDataParallel(model, device_ids=[0])
+                outputs = single_gpu_test(model, data_loader, args.show)
+            else:
+                model = MMDistributedDataParallel(model.cuda())
+                outputs = multi_gpu_test(model, data_loader, args.tmpdir)
+
+            rank, _ = get_dist_info()
+            if args.out and rank == 0:
+                eval_results_filename = (
+                    osp.splitext(args.out)[0] + '_results' +
+                    osp.splitext(args.out)[1])
+                mmcv.dump(outputs, args.out)
+                eval_types = args.eval
+                if cfg.dataset_type == 'VOCDataset':
+                    if eval_types:
+                        for eval_type in eval_types:
+                            if eval_type == 'bbox':
+                                test_dataset = mmcv.runner.obj_from_dict(
+                                    cfg.data.test, datasets)
+                                logger = 'print' if args.summaries else None
+                                mean_ap, eval_results = \
+                                    voc_eval_with_return(
+                                        args.out, test_dataset,
+                                        args.iou_thr, logger)
+                                aggregated_results[corruption][
+                                    corruption_severity] = eval_results
+                            else:
+                                print('\nOnly "bbox" evaluation \
+                                is supported for pascal voc')
+                else:
+                    if eval_types:
+                        print('Starting evaluate {}'.format(
+                            ' and '.join(eval_types)))
+                        if eval_types == ['proposal_fast']:
+                            result_file = args.out
+                        else:
+                            if not isinstance(outputs[0], dict):
+                                result_files = results2json(
+                                    dataset, outputs, args.out)
+                            else:
+                                for name in outputs[0]:
+                                    print('\nEvaluating {}'.format(name))
+                                    outputs_ = [out[name] for out in outputs]
+                                    result_file = args.out
+                                    + '.{}'.format(name)
+                                    result_files = results2json(
+                                        dataset, outputs_, result_file)
+                        eval_results = coco_eval_with_return(
+                            result_files, eval_types, dataset.coco)
+                        aggregated_results[corruption][
+                            corruption_severity] = eval_results
+                    else:
+                        print('\nNo task was selected for evaluation;'
+                              '\nUse --eval to select a task')
+
+            # save results after each evaluation
+            mmcv.dump(aggregated_results, eval_results_filename)
+
+    # print filan results
+    print('\nAggregated results:')
+    prints = args.final_prints
+    aggregate = args.final_prints_aggregate
+
+    if cfg.dataset_type == 'VOCDataset':
+        get_results(
+            eval_results_filename,
+            dataset='voc',
+            prints=prints,
+            aggregate=aggregate)
+    else:
+        get_results(
+            eval_results_filename,
+            dataset='coco',
+            prints=prints,
+            aggregate=aggregate)
+
+
+if __name__ == '__main__':
+    main()

tools/train.py

 from __future__ import division
 import argparse
 import os
+import os.path as osp
+import time
 
+import mmcv
 import torch
 from mmcv import Config
+from mmcv.runner import init_dist
 
 from mmdet import __version__
-from mmdet.apis import (get_root_logger, init_dist, set_random_seed,
-                        train_detector)
+from mmdet.apis import set_random_seed, train_detector
 from mmdet.datasets import build_dataset
 from mmdet.models import build_detector
-
+from mmdet.utils import get_root_logger
+import warnings
+warnings.filterwarnings("ignore")
 
 def parse_args():
     parser = argparse.ArgumentParser(description='Train a detector')
@@ -29,6 +34,10 @@ def parse_args():
         help='number of gpus to use '
         '(only applicable to non-distributed training)')
     parser.add_argument('--seed', type=int, default=None, help='random seed')
+    parser.add_argument(
+        '--deterministic',
+        action='store_true',
+        help='whether to set deterministic options for CUDNN backend.')
     parser.add_argument(
         '--launcher',
         choices=['none', 'pytorch', 'slurm', 'mpi'],
@@ -71,35 +80,46 @@ def main():
         distributed = True
         init_dist(args.launcher, **cfg.dist_params)
 
-    # init logger before other steps
-    logger = get_root_logger(cfg.log_level)
+    # create work_dir
+    mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir))
+    # init the logger before other steps
+    timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
+    log_file = osp.join(cfg.work_dir, '{}.log'.format(timestamp))
+    logger = get_root_logger(log_file=log_file, log_level=cfg.log_level)
+
+    # log some basic info
     logger.info('Distributed training: {}'.format(distributed))
+    logger.info('MMDetection Version: {}'.format(__version__))
+    logger.info('Config:\n{}'.format(cfg.text))
 
     # set random seeds
     if args.seed is not None:
-        logger.info('Set random seed to {}'.format(args.seed))
-        set_random_seed(args.seed)
+        logger.info('Set random seed to {}, deterministic: {}'.format(
+            args.seed, args.deterministic))
+        set_random_seed(args.seed, deterministic=args.deterministic)
 
     model = build_detector(
         cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)
 
-    train_dataset = build_dataset(cfg.data.train)
+    datasets = [build_dataset(cfg.data.train)]
+    if len(cfg.workflow) == 2:
+        datasets.append(build_dataset(cfg.data.val))
     if cfg.checkpoint_config is not None:
         # save mmdet version, config file content and class names in
         # checkpoints as meta data
         cfg.checkpoint_config.meta = dict(
             mmdet_version=__version__,
             config=cfg.text,
-            CLASSES=train_dataset.CLASSES)
+            CLASSES=datasets[0].CLASSES)
     # add an attribute for visualization convenience
-    model.CLASSES = train_dataset.CLASSES
+    model.CLASSES = datasets[0].CLASSES
     train_detector(
         model,
-        train_dataset,
+        datasets,
         cfg,
         distributed=distributed,
         validate=args.validate,
-        logger=logger)
+        timestamp=timestamp)
 
 
 if __name__ == '__main__':

tools/voc_eval.py

 from argparse import ArgumentParser
 
 import mmcv
-import numpy as np
 
 from mmdet import datasets
 from mmdet.core import eval_map
 
 
-def voc_eval(result_file, dataset, iou_thr=0.5):
+def voc_eval(result_file, dataset, iou_thr=0.5, nproc=4):
     det_results = mmcv.load(result_file)
-    gt_bboxes = []
-    gt_labels = []
-    gt_ignore = []
-    for i in range(len(dataset)):
-        ann = dataset.get_ann_info(i)
-        bboxes = ann['bboxes']
-        labels = ann['labels']
-        if 'bboxes_ignore' in ann:
-            ignore = np.concatenate([
-                np.zeros(bboxes.shape[0], dtype=np.bool),
-                np.ones(ann['bboxes_ignore'].shape[0], dtype=np.bool)
-            ])
-            gt_ignore.append(ignore)
-            bboxes = np.vstack([bboxes, ann['bboxes_ignore']])
-            labels = np.concatenate([labels, ann['labels_ignore']])
-        gt_bboxes.append(bboxes)
-        gt_labels.append(labels)
-    if not gt_ignore:
-        gt_ignore = gt_ignore
+    annotations = [dataset.get_ann_info(i) for i in range(len(dataset))]
     if hasattr(dataset, 'year') and dataset.year == 2007:
         dataset_name = 'voc07'
     else:
         dataset_name = dataset.CLASSES
     eval_map(
         det_results,
-        gt_bboxes,
-        gt_labels,
-        gt_ignore=gt_ignore,
+        annotations,
         scale_ranges=None,
         iou_thr=iou_thr,
         dataset=dataset_name,
-        print_summary=True)
+        logger='print',
+        nproc=nproc)
 
 
 def main():
@@ -52,10 +32,15 @@ def main():
         type=float,
         default=0.5,
         help='IoU threshold for evaluation')
+    parser.add_argument(
+        '--nproc',
+        type=int,
+        default=4,
+        help='Processes to be used for computing mAP')
     args = parser.parse_args()
     cfg = mmcv.Config.fromfile(args.config)
     test_dataset = mmcv.runner.obj_from_dict(cfg.data.test, datasets)
-    voc_eval(args.result, test_dataset, args.iou_thr)
+    voc_eval(args.result, test_dataset, args.iou_thr, args.nproc)
 
 
 if __name__ == '__main__':