diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4c6e5fa5730..b2e59d63ecd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
 #### Bugs
 
 * Fix #5580: [java-generator] Correctly handle defaults for IntOrString types
+* Fix #5584: Fix CRD generation when EnumMap is used
 
 #### Improvements
 * Fix #5429: moved crd generator annotations to generator-annotations instead of crd-generator-api. Using generator-annotations introduces no transitive dependencies.
diff --git a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AdditionalPrinterColumnDetector.java b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AdditionalPrinterColumnDetector.java
index 6eb7ebf5c72..d324801b9e1 100644
--- a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AdditionalPrinterColumnDetector.java
+++ b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AdditionalPrinterColumnDetector.java
@@ -17,8 +17,6 @@
 
 import io.fabric8.crd.generator.annotation.PrinterColumn;
 
-import java.util.ArrayList;
-
 public class AdditionalPrinterColumnDetector extends AnnotatedMultiPropertyPathDetector {
 
   public AdditionalPrinterColumnDetector() {
@@ -26,6 +24,6 @@ public AdditionalPrinterColumnDetector() {
   }
 
   public AdditionalPrinterColumnDetector(String prefix) {
-    super(prefix, PrinterColumn.class.getSimpleName(), new ArrayList<>());
+    super(prefix, PrinterColumn.class.getSimpleName());
   }
 }
diff --git a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AnnotatedMultiPropertyPathDetector.java b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AnnotatedMultiPropertyPathDetector.java
index 2c5d38b83eb..57eac329bbd 100644
--- a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AnnotatedMultiPropertyPathDetector.java
+++ b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AnnotatedMultiPropertyPathDetector.java
@@ -22,7 +22,9 @@
 import io.sundr.model.TypeDef;
 import io.sundr.model.TypeDefBuilder;
 
+import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Deque;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -40,21 +42,19 @@ public class AnnotatedMultiPropertyPathDetector extends TypedVisitor<TypeDefBuil
   private final String annotationName;
   private final List<Property> parents;
   private final Map<String, Property> properties;
+  private final Deque<Runnable> toRun;
 
   public AnnotatedMultiPropertyPathDetector(String prefix, String annotationName) {
-    this(prefix, annotationName, new ArrayList<>());
-  }
-
-  public AnnotatedMultiPropertyPathDetector(String prefix, String annotationName, List<Property> parents) {
-    this(prefix, annotationName, parents, new HashMap<>());
+    this(prefix, annotationName, new ArrayList<>(), new HashMap<>(), new ArrayDeque<>());
   }
 
   public AnnotatedMultiPropertyPathDetector(String prefix, String annotationName, List<Property> parents,
-      Map<String, Property> properties) {
+      Map<String, Property> properties, Deque<Runnable> toRun) {
     this.prefix = prefix;
     this.annotationName = annotationName;
     this.parents = parents;
     this.properties = properties;
+    this.toRun = toRun;
   }
 
   private boolean excludePropertyProcessing(Property p) {
@@ -84,15 +84,20 @@ public void visit(TypeDefBuilder builder) {
       if (!parents.contains(p) && !excludePropertyProcessing(p)) {
         ClassRef classRef = (ClassRef) p.getTypeRef();
         TypeDef propertyType = Types.typeDefFrom(classRef);
-        if (!propertyType.isEnum()) {
+        if (!propertyType.isEnum() && !classRef.getPackageName().startsWith("java.")) {
           List<Property> newParents = new ArrayList<>(parents);
           newParents.add(p);
-          new TypeDefBuilder(propertyType)
-              .accept(new AnnotatedMultiPropertyPathDetector(prefix, annotationName, newParents, properties))
-              .build();
+          toRun.add(() -> new TypeDefBuilder(propertyType)
+              .accept(new AnnotatedMultiPropertyPathDetector(prefix, annotationName, newParents, properties, toRun)));
         }
       }
     });
+
+    if (parents.isEmpty()) {
+      while (!toRun.isEmpty()) {
+        toRun.pop().run();
+      }
+    }
   }
 
   public Set<String> getPaths() {
diff --git a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AnnotatedPropertyPathDetector.java b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AnnotatedPropertyPathDetector.java
index 5ae2ee8c222..8cd191abc8f 100644
--- a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AnnotatedPropertyPathDetector.java
+++ b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/AnnotatedPropertyPathDetector.java
@@ -23,7 +23,9 @@
 import io.sundr.model.TypeDef;
 import io.sundr.model.TypeDefBuilder;
 
+import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Deque;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicReference;
@@ -39,22 +41,20 @@ public class AnnotatedPropertyPathDetector extends TypedVisitor<TypeDefBuilder>
   private final String prefix;
   private final String annotationName;
   private final List<Property> parents;
-  private final AtomicReference<Optional<String>> reference;
+  private final AtomicReference<String> reference;
+  private final Deque<Runnable> toRun;
 
   public AnnotatedPropertyPathDetector(String prefix, String annotationName) {
-    this(prefix, annotationName, new ArrayList<>());
-  }
-
-  public AnnotatedPropertyPathDetector(String prefix, String annotationName, List<Property> parents) {
-    this(prefix, annotationName, parents, new AtomicReference<>(Optional.empty()));
+    this(prefix, annotationName, new ArrayList<>(), new AtomicReference<>(), new ArrayDeque<>());
   }
 
   public AnnotatedPropertyPathDetector(String prefix, String annotationName, List<Property> parents,
-      AtomicReference<Optional<String>> reference) {
+      AtomicReference<String> reference, Deque<Runnable> toRun) {
     this.prefix = prefix;
     this.annotationName = annotationName;
     this.parents = parents;
     this.reference = reference;
+    this.toRun = toRun;
   }
 
   private static boolean excludePropertyProcessing(Property p) {
@@ -70,32 +70,10 @@ private static boolean excludePropertyProcessing(Property p) {
   public void visit(TypeDefBuilder builder) {
     TypeDef type = builder.build();
     final List<Property> properties = type.getProperties();
-    if (visitProperties(properties)) {
-      return;
-    }
-    visitPropertiesClasses(properties);
-  }
-
-  private void visitPropertiesClasses(List<Property> properties) {
-    for (Property p : properties) {
-      if (!(p.getTypeRef() instanceof ClassRef)) {
-        continue;
-      }
-      if (!parents.contains(p) && !excludePropertyProcessing(p)) {
-        ClassRef classRef = (ClassRef) p.getTypeRef();
-        TypeDef propertyType = Types.typeDefFrom(classRef);
-        if (!propertyType.isEnum()) {
-          List<Property> newParents = new ArrayList<>(parents);
-          newParents.add(p);
-          new TypeDefBuilder(propertyType)
-              .accept(new AnnotatedPropertyPathDetector(prefix, annotationName, newParents, reference))
-              .build();
-        }
-      }
-    }
+    visitProperties(properties);
   }
 
-  private boolean visitProperties(List<Property> properties) {
+  private void visitProperties(List<Property> properties) {
     for (Property p : properties) {
       if (parents.contains(p)) {
         continue;
@@ -107,15 +85,30 @@ private boolean visitProperties(List<Property> properties) {
         match = annotation.getClassRef().getName().equals(annotationName);
         if (match) {
           newParents.add(p);
-          reference.set(Optional.of(newParents.stream().map(Property::getName).collect(Collectors.joining(DOT, prefix, ""))));
-          return true;
+          reference.set(newParents.stream().map(Property::getName).collect(Collectors.joining(DOT, prefix, "")));
+          return;
+        }
+      }
+
+      if (p.getTypeRef() instanceof ClassRef && !excludePropertyProcessing(p)) {
+        ClassRef classRef = (ClassRef) p.getTypeRef();
+        TypeDef propertyType = Types.typeDefFrom(classRef);
+        if (!propertyType.isEnum() && !classRef.getPackageName().startsWith("java.")) {
+          newParents.add(p);
+          new TypeDefBuilder(propertyType)
+              .accept(new AnnotatedPropertyPathDetector(prefix, annotationName, newParents, reference, toRun));
         }
       }
     }
-    return false;
+
+    if (parents.isEmpty()) {
+      while (!toRun.isEmpty() && reference.get() == null) {
+        toRun.pop().run();
+      }
+    }
   }
 
   public Optional<String> getPath() {
-    return reference.get();
+    return Optional.ofNullable(reference.get());
   }
 }
diff --git a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/LabelSelectorPathDetector.java b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/LabelSelectorPathDetector.java
index dda802865e9..ff4609c084f 100644
--- a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/LabelSelectorPathDetector.java
+++ b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/LabelSelectorPathDetector.java
@@ -17,8 +17,6 @@
 
 import io.fabric8.kubernetes.model.annotation.LabelSelector;
 
-import java.util.ArrayList;
-
 public class LabelSelectorPathDetector extends AnnotatedPropertyPathDetector {
 
   public LabelSelectorPathDetector() {
@@ -26,6 +24,6 @@ public LabelSelectorPathDetector() {
   }
 
   public LabelSelectorPathDetector(String prefix) {
-    super(prefix, LabelSelector.class.getSimpleName(), new ArrayList<>());
+    super(prefix, LabelSelector.class.getSimpleName());
   }
 }
diff --git a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/SpecReplicasPathDetector.java b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/SpecReplicasPathDetector.java
index 74d2ac3b758..9513166ec09 100644
--- a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/SpecReplicasPathDetector.java
+++ b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/SpecReplicasPathDetector.java
@@ -17,8 +17,6 @@
 
 import io.fabric8.kubernetes.model.annotation.SpecReplicas;
 
-import java.util.ArrayList;
-
 public class SpecReplicasPathDetector extends AnnotatedPropertyPathDetector {
 
   public SpecReplicasPathDetector() {
@@ -26,6 +24,6 @@ public SpecReplicasPathDetector() {
   }
 
   public SpecReplicasPathDetector(String prefix) {
-    super(prefix, SpecReplicas.class.getSimpleName(), new ArrayList<>());
+    super(prefix, SpecReplicas.class.getSimpleName());
   }
 }
diff --git a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/StatusReplicasPathDetector.java b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/StatusReplicasPathDetector.java
index f982bdf1166..a573cbeffc7 100644
--- a/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/StatusReplicasPathDetector.java
+++ b/crd-generator/api/src/main/java/io/fabric8/crd/generator/visitor/StatusReplicasPathDetector.java
@@ -17,12 +17,10 @@
 
 import io.fabric8.kubernetes.model.annotation.StatusReplicas;
 
-import java.util.ArrayList;
-
 public class StatusReplicasPathDetector extends AnnotatedPropertyPathDetector {
 
   public StatusReplicasPathDetector(String prefix) {
-    super(prefix, StatusReplicas.class.getSimpleName(), new ArrayList<>());
+    super(prefix, StatusReplicas.class.getSimpleName());
 
   }
 
diff --git a/crd-generator/api/src/test/java/io/fabric8/crd/example/map/ContainingMaps.java b/crd-generator/api/src/test/java/io/fabric8/crd/example/map/ContainingMaps.java
index 0bc432d06fe..9f80b3a0da8 100644
--- a/crd-generator/api/src/test/java/io/fabric8/crd/example/map/ContainingMaps.java
+++ b/crd-generator/api/src/test/java/io/fabric8/crd/example/map/ContainingMaps.java
@@ -19,8 +19,16 @@
 import io.fabric8.kubernetes.model.annotation.Group;
 import io.fabric8.kubernetes.model.annotation.Version;
 
+import java.util.EnumMap;
+
 @Group("map.fabric8.io")
 @Version("v1alpha1")
 public class ContainingMaps extends CustomResource<ContainingMapsSpec, Void> {
 
+  public enum Foo {
+    BAR
+  }
+
+  private EnumMap<Foo, String> enumToStringMap;
+
 }