diff --git a/mmdet/models/bbox_heads/bbox_head.py b/mmdet/models/bbox_heads/bbox_head.py
index 6188fb8eaf46ed29c731d35c6fb06444be232d56..2e983ff3c2810ef3a9803dd4c8b66980b0751ba3 100644
--- a/mmdet/models/bbox_heads/bbox_head.py
+++ b/mmdet/models/bbox_heads/bbox_head.py
@@ -155,7 +155,9 @@ class BBoxHead(nn.Module):
             if isinstance(scale_factor, float):
                 bboxes /= scale_factor
             else:
-                bboxes /= torch.from_numpy(scale_factor).to(bboxes.device)
+                scale_factor = torch.from_numpy(scale_factor).to(bboxes.device)
+                bboxes = (bboxes.view(bboxes.size(0), -1, 4) /
+                          scale_factor).view(bboxes.size()[0], -1)
 
         if cfg is None:
             return bboxes, scores
diff --git a/mmdet/models/detectors/test_mixins.py b/mmdet/models/detectors/test_mixins.py
index 1abd6094644bb5ee2779401c8ed31525cf997903..05b2081271e09ed9c845c593cbe4946a5ab31e97 100644
--- a/mmdet/models/detectors/test_mixins.py
+++ b/mmdet/models/detectors/test_mixins.py
@@ -1,3 +1,5 @@
+import torch
+
 from mmdet.core import (bbox2roi, bbox_mapping, merge_aug_bboxes,
                         merge_aug_masks, merge_aug_proposals, multiclass_nms)
 
@@ -114,6 +116,9 @@ class MaskTestMixin(object):
         else:
             # if det_bboxes is rescaled to the original image size, we need to
             # rescale it back to the testing scale to obtain RoIs.
+            if rescale and not isinstance(scale_factor, float):
+                scale_factor = torch.from_numpy(scale_factor).to(
+                    det_bboxes.device)
             _bboxes = (
                 det_bboxes[:, :4] * scale_factor if rescale else det_bboxes)
             mask_rois = bbox2roi([_bboxes])
diff --git a/mmdet/models/mask_heads/fcn_mask_head.py b/mmdet/models/mask_heads/fcn_mask_head.py
index f25aa172855bc8c0f8f29de60c6ea711714108de..f9dafcd7e69e71b70c2161de76b87e07fa5ca9d2 100644
--- a/mmdet/models/mask_heads/fcn_mask_head.py
+++ b/mmdet/models/mask_heads/fcn_mask_head.py
@@ -159,6 +159,8 @@ class FCNMaskHead(nn.Module):
             scale_factor = 1.0
 
         for i in range(bboxes.shape[0]):
+            if not isinstance(scale_factor, (float, np.ndarray)):
+                scale_factor = scale_factor.cpu().numpy()
             bbox = (bboxes[i, :] / scale_factor).astype(np.int32)
             label = labels[i]
             w = max(bbox[2] - bbox[0] + 1, 1)