diff --git a/topi/python/topi/x86/conv2d.py b/topi/python/topi/x86/conv2d.py
index 316be188e348d..5775d80f92428 100644
--- a/topi/python/topi/x86/conv2d.py
+++ b/topi/python/topi/x86/conv2d.py
@@ -419,13 +419,14 @@ def _alter_conv2d_layout(attrs, inputs, tinfo, F):
     # only optimize for NCHW
     if layout != 'NCHW':
         return None
-    if groups != 1 and not is_depthwise:
-        return None
 
     assert attrs["kernel_layout"] == "OIHW"
     kshape = get_const_tuple(kernel.shape)
     is_depthwise = groups == kshape[0] and kshape[1] == 1
 
+    if groups != 1 and not is_depthwise:
+        return None
+
     dispatch_ctx = autotvm.task.DispatchContext.current
     target = tvm.target.current_target()
     # query schedule and fallback if necessary