diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupportedUsageTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupportedUsageTest.java
index 7bdb8dfc8aa7..879e790206c3 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupportedUsageTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupportedUsageTest.java
@@ -22,7 +22,8 @@
 import org.junit.Test;
 import org.reflections.Reflections;
 import org.reflections.scanners.MethodAnnotationsScanner;
-
+import org.reflections.util.ClasspathHelper;
+import org.reflections.util.ConfigurationBuilder;
 import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.Collections;
@@ -36,15 +37,45 @@ public class NotYetSupportedUsageTest
   @Test
   public void ensureAllModesUsed()
   {
-    Set<Method> methodsAnnotatedWith = new Reflections("org.apache", new MethodAnnotationsScanner())
-        .getMethodsAnnotatedWith(NotYetSupported.class);
+    if (false) {
+      ConfigurationBuilder configBuilder = new ConfigurationBuilder();
+      configBuilder = configBuilder.setScanners(new MethodAnnotationsScanner());
+//      if (true) {
+//        configBuilder = configBuilder
+//            .filterInputsBy(new FilterBuilder().include(FilterBuilder.prefix("org.apache.druid.sql")));
+//      }
+//      if (true) {
+//        configBuilder = configBuilder.filterInputsBy(FilterBuilder.parse("+.*Test.*"));
+//      }
+      if (false) {
+        configBuilder = configBuilder.setUrls(ClasspathHelper.forPackage("org.apache.druid"));
+      } else {
+        configBuilder = configBuilder.setUrls(ClasspathHelper.forClass(NotYetSupported.class));
+      }
 
-    Set<NotYetSupported.Modes> modes = new HashSet<>(Arrays.asList(NotYetSupported.Modes.values()));
-    for (Method method : methodsAnnotatedWith) {
-      NotYetSupported annot = method.getAnnotation(NotYetSupported.class);
-      modes.remove(annot.value());
-    }
+      Set<Method> methodsAnnotatedWith = new Reflections(configBuilder)
+          .getMethodsAnnotatedWith(NotYetSupported.class);
+
+      System.out.println(methodsAnnotatedWith);
+      Set<NotYetSupported.Modes> modes = new HashSet<>(Arrays.asList(NotYetSupported.Modes.values()));
+      for (Method method : methodsAnnotatedWith) {
+        NotYetSupported annot = method.getAnnotation(NotYetSupported.class);
+        modes.remove(annot.value());
+      }
+
+      assertEquals("There are unused modes which should be removed", Collections.emptySet(), modes);
+    } else {
 
-    assertEquals("There are unused modes which should be removed", Collections.emptySet(), modes);
+      Set<Method> methodsAnnotatedWith = new Reflections("org.apache", new MethodAnnotationsScanner())
+          .getMethodsAnnotatedWith(NotYetSupported.class);
+
+      Set<NotYetSupported.Modes> modes = new HashSet<>(Arrays.asList(NotYetSupported.Modes.values()));
+      for (Method method : methodsAnnotatedWith) {
+        NotYetSupported annot = method.getAnnotation(NotYetSupported.class);
+        modes.remove(annot.value());
+      }
+
+      assertEquals("There are unused modes which should be removed", Collections.emptySet(), modes);
+    }
   }
 }