diff --git a/torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp b/torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp
index f89e6cc3030..480cda9bf51 100644
--- a/torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp
+++ b/torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp
@@ -244,7 +244,8 @@ void deformable_im2col(
 }
 
 int get_greatest_divisor_below_bound(int n, int bound) {
-  for (int k = bound; k > 1; --k) {
+  int limit = std::min(n / 2, bound);
+  for (int k = limit; k > 1; --k) {
     if (n % k == 0) {
       return k;
     }
diff --git a/torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu b/torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu
index ae496b37d48..cc3a6d42739 100644
--- a/torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu
+++ b/torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu
@@ -305,7 +305,8 @@ void deformable_im2col(
 }
 
 int get_greatest_divisor_below_bound(int n, int bound) {
-  for (int k = bound; k > 1; --k) {
+  int limit = std::min(n / 2, bound);
+  for (int k = limit; k > 1; --k) {
     if (n % k == 0) {
       return k;
     }