diff --git a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/AbstractCustomResourceHandler.java b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/AbstractCustomResourceHandler.java index 0e60ed2502c..1f5e4b20fb4 100644 --- a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/AbstractCustomResourceHandler.java +++ b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/AbstractCustomResourceHandler.java @@ -25,7 +25,7 @@ */ public abstract class AbstractCustomResourceHandler { - public abstract void handle(CustomResourceInfo config); + public abstract void handle(CustomResourceInfo config, ResolvingContext resolvingContext); public abstract Stream finish(); diff --git a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/AbstractJsonSchema.java b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/AbstractJsonSchema.java index 6e1236c0c44..608861a1f4f 100644 --- a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/AbstractJsonSchema.java +++ b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/AbstractJsonSchema.java @@ -145,7 +145,7 @@ private T resolveRoot(Class definition) { return resolveObject(new LinkedHashMap<>(), schemaSwaps, schema, "kind", "apiVersion", "metadata"); } return resolveProperty(new LinkedHashMap<>(), schemaSwaps, null, - resolvingContext.serializationConfig.constructType(definition), schema); + resolvingContext.objectMapper.getSerializationConfig().constructType(definition), schema); } /** @@ -282,9 +282,12 @@ private T resolveObject(LinkedHashMap visited, InternalSchemaSwa final InternalSchemaSwaps swaps = schemaSwaps; GeneratorObjectSchema gos = (GeneratorObjectSchema) jacksonSchema.asObjectSchema(); - AnnotationIntrospector ai = resolvingContext.serializationConfig.getAnnotationIntrospector(); - BeanDescription bd = resolvingContext.serializationConfig.introspect(gos.javaType); - boolean preserveUnknownFields = bd.findAnyGetter() != null || bd.findAnySetterAccessor() != null; + AnnotationIntrospector ai = resolvingContext.objectMapper.getSerializationConfig().getAnnotationIntrospector(); + BeanDescription bd = resolvingContext.objectMapper.getSerializationConfig().introspect(gos.javaType); + boolean preserveUnknownFields = false; + if (resolvingContext.implicitPreserveUnknownFields) { + preserveUnknownFields = bd.findAnyGetter() != null || bd.findAnySetterAccessor() != null; + } consumeRepeatingAnnotation(gos.javaType.getRawClass(), SchemaSwap.class, ss -> { swaps.registerSwap(gos.javaType.getRawClass(), @@ -333,7 +336,7 @@ private T resolveObject(LinkedHashMap visited, InternalSchemaSwa continue; } propertySchema = toJsonSchema(propertyMetadata.schemaFrom); - type = resolvingContext.serializationConfig.constructType(propertyMetadata.schemaFrom); + type = resolvingContext.objectMapper.getSerializationConfig().constructType(propertyMetadata.schemaFrom); } T schema = resolveProperty(visited, schemaSwaps, name, type, propertySchema); @@ -423,7 +426,7 @@ private T resolveProperty(LinkedHashMap visited, InternalSchemaS } else if (jacksonSchema instanceof ReferenceSchema) { // de-reference the reference schema - these can be naturally non-cyclic, for example siblings ReferenceSchema ref = (ReferenceSchema) jacksonSchema; - GeneratorObjectSchema referenced = resolvingContext.seen.get(ref.get$ref()); + GeneratorObjectSchema referenced = resolvingContext.uriToJacksonSchema.get(ref.get$ref()); Utils.checkNotNull(referenced, "Could not find previously generated schema"); jacksonSchema = referenced; } else if (type.isMapLikeType()) { @@ -469,11 +472,10 @@ private Set findIngoredEnumConstants(JavaType type) { Set toIgnore = new HashSet<>(); for (Field field : fields) { if (field.isEnumConstant() && field.getAnnotation(JsonIgnore.class) != null) { - // hack to figure out the enum constant + // hack to figure out the enum constant - this guards against some using both JsonIgnore and JsonProperty try { Object value = field.get(null); - toIgnore.add(resolvingContext.kubernetesSerialization - .unmarshal(resolvingContext.kubernetesSerialization.asJson(value), String.class)); + toIgnore.add(resolvingContext.objectMapper.convertValue(value, String.class)); } catch (IllegalArgumentException | IllegalAccessException e) { } } diff --git a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/CRDGenerator.java b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/CRDGenerator.java index d537f5d24cd..e2a4042ec51 100644 --- a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/CRDGenerator.java +++ b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/CRDGenerator.java @@ -41,6 +41,9 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinTask; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY; @@ -53,6 +56,8 @@ public class CRDGenerator { private final Map handlers = new HashMap<>(2); private CRDOutput output; private boolean parallel; + private boolean implicitPreserveUnknownFields; + private ObjectMapper objectMapper; private Map infos; // TODO: why not rely on the standard fabric8 yaml mapping @@ -77,11 +82,21 @@ public CRDGenerator withOutput(CRDOutput output) { return this; } + public CRDGenerator withImplicitPreserveUnknownFields(boolean implicitPreserveUnknownFields) { + this.implicitPreserveUnknownFields = implicitPreserveUnknownFields; + return this; + } + public CRDGenerator withParallelGenerationEnabled(boolean parallel) { this.parallel = parallel; return this; } + public CRDGenerator withObjectMapper(ObjectMapper mapper) { + this.objectMapper = mapper; + return this; + } + public CRDGenerator forCRDVersions(List versions) { return versions != null && !versions.isEmpty() ? forCRDVersions(versions.toArray(new String[0])) : this; @@ -154,6 +169,13 @@ public CRDGenerationInfo detailedGenerate() { forCRDVersions(CustomResourceHandler.VERSION); } + final ResolvingContext context; + if (this.objectMapper == null) { + context = ResolvingContext.defaultResolvingContext(implicitPreserveUnknownFields); + } else { + context = new ResolvingContext(this.objectMapper, implicitPreserveUnknownFields); + } + for (CustomResourceInfo info : infos.values()) { if (info != null) { if (LOGGER.isInfoEnabled()) { @@ -162,7 +184,13 @@ public CRDGenerationInfo detailedGenerate() { info.specClassName().orElse("undetermined"), info.statusClassName().orElse("undetermined")); } - handlers.values().forEach(h -> h.handle(info)); + + if (parallel) { + handlers.values().stream().map(h -> ForkJoinPool.commonPool().submit(() -> h.handle(info, context.forkContext()))) + .collect(Collectors.toList()).stream().forEach(ForkJoinTask::join); + } else { + handlers.values().stream().forEach(h -> h.handle(info, context)); + } } } diff --git a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/ResolvingContext.java b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/ResolvingContext.java index 5c8b9572d0a..df7fccaf561 100644 --- a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/ResolvingContext.java +++ b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/ResolvingContext.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationConfig; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonObjectFormatVisitor; import com.fasterxml.jackson.module.jsonSchema.JsonSchema; @@ -31,10 +30,15 @@ import com.fasterxml.jackson.module.jsonSchema.types.ObjectSchema; import io.fabric8.kubernetes.client.utils.KubernetesSerialization; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +/** + * Encapsulates the stateful Jackson details that allow for crd to be fully resolved by our logic + * - holds an association of uris to already generated jackson schemas + * - holds a Jackson SchemaGenerator which is not thread-safe + */ public class ResolvingContext { static final class GeneratorObjectSchema extends ObjectSchema { @@ -76,15 +80,15 @@ public JsonObjectFormatVisitor expectObjectFormat(JavaType convertedType) { // so that we may directly mark preserve unknown JsonObjectFormatVisitor result = super.expectObjectFormat(convertedType); ((GeneratorObjectSchema) schema).javaType = convertedType; - seen.putIfAbsent(this.visitorContext.getSeenSchemaUri(convertedType), (GeneratorObjectSchema) schema); + uriToJacksonSchema.putIfAbsent(this.visitorContext.getSeenSchemaUri(convertedType), (GeneratorObjectSchema) schema); return result; } } final JsonSchemaGenerator generator; - final SerializationConfig serializationConfig; - final KubernetesSerialization kubernetesSerialization; - final Map seen = new HashMap<>(); + final ObjectMapper objectMapper; + final Map uriToJacksonSchema; + final boolean implicitPreserveUnknownFields; private static class AccessibleKubernetesSerialization extends KubernetesSerialization { @@ -97,16 +101,26 @@ public ObjectMapper getMapper() { private static AccessibleKubernetesSerialization DEFAULT_KUBERNETES_SERIALIZATION; - public static ResolvingContext defaultResolvingContext() { + public static ResolvingContext defaultResolvingContext(boolean implicitPreserveUnknownFields) { if (DEFAULT_KUBERNETES_SERIALIZATION == null) { DEFAULT_KUBERNETES_SERIALIZATION = new AccessibleKubernetesSerialization(); } - return new ResolvingContext(DEFAULT_KUBERNETES_SERIALIZATION.getMapper(), DEFAULT_KUBERNETES_SERIALIZATION); + return new ResolvingContext(DEFAULT_KUBERNETES_SERIALIZATION.getMapper(), implicitPreserveUnknownFields); + } + + public ResolvingContext forkContext() { + return new ResolvingContext(objectMapper, uriToJacksonSchema, implicitPreserveUnknownFields); + } + + public ResolvingContext(ObjectMapper mapper, boolean implicitPreserveUnknownFields) { + this(mapper, new ConcurrentHashMap<>(), implicitPreserveUnknownFields); } - public ResolvingContext(ObjectMapper mapper, KubernetesSerialization kubernetesSerialization) { - serializationConfig = mapper.getSerializationConfig(); - this.kubernetesSerialization = kubernetesSerialization; + private ResolvingContext(ObjectMapper mapper, Map uriToJacksonSchema, + boolean implicitPreserveUnknownFields) { + this.uriToJacksonSchema = uriToJacksonSchema; + this.objectMapper = mapper; + this.implicitPreserveUnknownFields = implicitPreserveUnknownFields; generator = new JsonSchemaGenerator(mapper, new WrapperFactory() { @Override diff --git a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/v1/CustomResourceHandler.java b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/v1/CustomResourceHandler.java index cb46efa4407..46f9cf50418 100644 --- a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/v1/CustomResourceHandler.java +++ b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/v1/CustomResourceHandler.java @@ -35,23 +35,24 @@ import java.util.List; import java.util.Optional; +import java.util.Queue; import java.util.TreeMap; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.stream.Collectors; import java.util.stream.Stream; public class CustomResourceHandler extends AbstractCustomResourceHandler { - private List crds = new CopyOnWriteArrayList<>(); + private Queue crds = new ConcurrentLinkedQueue<>(); public static final String VERSION = "v1"; @Override - public void handle(CustomResourceInfo config) { + public void handle(CustomResourceInfo config, ResolvingContext resolvingContext) { final String name = config.crdName(); final String version = config.version(); - JsonSchema resolver = new JsonSchema(ResolvingContext.defaultResolvingContext(), config.definition()); + JsonSchema resolver = new JsonSchema(resolvingContext, config.definition()); JSONSchemaProps schema = resolver.getSchema(); CustomResourceDefinitionVersionBuilder builder = new CustomResourceDefinitionVersionBuilder() @@ -146,16 +147,16 @@ private CustomResourceDefinition combine(List definiti .map(CustomResourceDefinitionVersion::getName) .collect(Collectors.toList()); - if (storageVersions.size() > 1) { + if (storageVersions.size() > 1) { throw new IllegalStateException(String.format( "'%s' custom resource has versions %s marked as storage. Only one version can be marked as storage per custom resource.", primary.getMetadata().getName(), storageVersions)); - } + } - versions = KubernetesVersionPriority.sortByPriority(versions, CustomResourceDefinitionVersion::getName); + versions = KubernetesVersionPriority.sortByPriority(versions, CustomResourceDefinitionVersion::getName); - //TODO: we could double check that the top-level metadata is consistent across all versions - return new CustomResourceDefinitionBuilder(primary).editSpec().withVersions(versions).endSpec().build(); + //TODO: we could double check that the top-level metadata is consistent across all versions + return new CustomResourceDefinitionBuilder(primary).editSpec().withVersions(versions).endSpec().build(); } } diff --git a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/v1/JsonSchema.java b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/v1/JsonSchema.java index b5fa2c60167..e845d4c89c2 100644 --- a/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/v1/JsonSchema.java +++ b/crd-generator/api-v2/src/main/java/io/fabric8/crdv2/generator/v1/JsonSchema.java @@ -41,7 +41,7 @@ public static class V1JSONSchemaProps extends JSONSchemaProps implements Kuberne } public static JSONSchemaProps from(Class definition) { - return new JsonSchema(ResolvingContext.defaultResolvingContext(), definition).getSchema(); + return new JsonSchema(ResolvingContext.defaultResolvingContext(false), definition).getSchema(); } public JsonSchema(ResolvingContext resolvingContext, Class definition) { diff --git a/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/example/annotated/AnnotatedSpec.java b/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/example/annotated/AnnotatedSpec.java index 39b77d0a5fd..55cfe8837ab 100644 --- a/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/example/annotated/AnnotatedSpec.java +++ b/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/example/annotated/AnnotatedSpec.java @@ -151,6 +151,7 @@ public enum AnnotatedEnum { non("N"), @JsonProperty("oui") es("O"), + @JsonProperty("foo") @JsonIgnore Maybe("Maybe"); diff --git a/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/generator/v1/JsonSchemaTest.java b/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/generator/v1/JsonSchemaTest.java index ad00af91735..ac1327e8969 100644 --- a/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/generator/v1/JsonSchemaTest.java +++ b/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/generator/v1/JsonSchemaTest.java @@ -38,6 +38,7 @@ import io.fabric8.crdv2.example.extraction.NestedSchemaSwap; import io.fabric8.crdv2.example.json.ContainingJson; import io.fabric8.crdv2.example.person.Person; +import io.fabric8.crdv2.generator.ResolvingContext; import io.fabric8.kubernetes.api.model.AnyType; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Quantity; @@ -188,7 +189,7 @@ void shouldAugmentPropertiesSchemaFromAnnotations() throws JsonProcessingExcepti @Test void shouldProduceKubernetesPreserveFields() { - JSONSchemaProps schema = JsonSchema.from(ContainingJson.class); + JSONSchemaProps schema = new JsonSchema(ResolvingContext.defaultResolvingContext(true), ContainingJson.class).getSchema(); assertNotNull(schema); Map properties = assertSchemaHasNumberOfProperties(schema, 2); final JSONSchemaProps specSchema = properties.get("spec"); @@ -344,7 +345,7 @@ public void setValues(Map values) { @Test void testPreserveUnknown() { - JSONSchemaProps schema = JsonSchema.from(PreserveUnknown.class); + JSONSchemaProps schema = new JsonSchema(ResolvingContext.defaultResolvingContext(true), PreserveUnknown.class).getSchema(); assertNotNull(schema); assertEquals(0, schema.getProperties().size()); assertEquals(Boolean.TRUE, schema.getXKubernetesPreserveUnknownFields()); diff --git a/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/generator/v1/SpecReplicasPathTest.java b/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/generator/v1/SpecReplicasPathTest.java index 095d80b28ea..411c97cb341 100644 --- a/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/generator/v1/SpecReplicasPathTest.java +++ b/crd-generator/api-v2/src/test/java/io/fabric8/crdv2/generator/v1/SpecReplicasPathTest.java @@ -30,7 +30,7 @@ class SpecReplicasPathTest { @Test public void shoudDetectSpecReplicasPath() throws Exception { - JsonSchema resolver = new JsonSchema(ResolvingContext.defaultResolvingContext(), WebServerWithStatusProperty.class); + JsonSchema resolver = new JsonSchema(ResolvingContext.defaultResolvingContext(false), WebServerWithStatusProperty.class); Optional path = resolver.getSinglePath(SpecReplicas.class); assertTrue(path.isPresent()); assertEquals(".replicas", path.get()); @@ -38,7 +38,7 @@ public void shoudDetectSpecReplicasPath() throws Exception { @Test public void shoudDetectNestedSpecReplicasPath() throws Exception { - JsonSchema resolver = new JsonSchema(ResolvingContext.defaultResolvingContext(), WebServerWithSpec.class); + JsonSchema resolver = new JsonSchema(ResolvingContext.defaultResolvingContext(false), WebServerWithSpec.class); Optional path = resolver.getSinglePath(SpecReplicas.class); assertTrue(path.isPresent()); assertEquals(".spec.replicas", path.get()); diff --git a/crd-generator/test/pom.xml b/crd-generator/test/pom.xml index f667ad46976..96240459e9e 100644 --- a/crd-generator/test/pom.xml +++ b/crd-generator/test/pom.xml @@ -38,12 +38,7 @@ io.fabric8 crd-generator-apt - - - io.fabric8 - crd-generator-api-v2 - - + org.projectlombok lombok @@ -81,13 +76,6 @@ ${keycloak-version} test - - - org.jboss.logging - jboss-logging - 3.5.3.Final - test - diff --git a/crd-generator/test/src/test/java/io/fabric8/crd/generator/keycloak/KeycloakRealmCRDTest.java b/crd-generator/test/src/test/java/io/fabric8/crd/generator/keycloak/KeycloakRealmCRDTest.java index a9f615e6de1..881c1c36c48 100644 --- a/crd-generator/test/src/test/java/io/fabric8/crd/generator/keycloak/KeycloakRealmCRDTest.java +++ b/crd-generator/test/src/test/java/io/fabric8/crd/generator/keycloak/KeycloakRealmCRDTest.java @@ -15,46 +15,19 @@ */ package io.fabric8.crd.generator.keycloak; -import io.fabric8.crdv2.generator.CRDGenerator; -import io.fabric8.crdv2.generator.CRDGenerator.AbstractCRDOutput; import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; -import io.fabric8.kubernetes.client.utils.IOHelpers; import io.fabric8.kubernetes.client.utils.Serialization; import org.junit.jupiter.api.Test; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; class KeycloakRealmCRDTest { @Test - void testCrd() throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - CRDGenerator generator = new CRDGenerator().withOutput(new AbstractCRDOutput() { - - @Override - protected OutputStream createStreamFor(String crdName) throws IOException { - return baos; - } - - }); - generator.customResourceClasses(KeycloakRealm.class); - generator.generate(); - String result = baos.toString(); - - CustomResourceDefinition d1 = Serialization.unmarshal(result, - CustomResourceDefinition.class); - - CustomResourceDefinition d2 = Serialization.unmarshal(getClass().getClassLoader() + void testCrd() { + CustomResourceDefinition d = Serialization.unmarshal(getClass().getClassLoader() .getResourceAsStream("META-INF/fabric8/keycloakrealms.sample.fabric8.io-v1.yml"), CustomResourceDefinition.class); - - assertEquals(IOHelpers.readFully(getClass().getClassLoader() - .getResourceAsStream("META-INF/fabric8/keycloakrealms.sample.fabric8.io-v1.yml")), result); - - //assertNotNull(d); // just make sure it generates + assertNotNull(d); // just make sure it generates } }