XSD import fix (finos#865)

* Fixed name clashes with new config file * Fixed annotation name clash * Escaped Object type * Fix XML config * Added support for substitution groups * Enum xml config fix * Enum xml config fix * Casing config * Update expectation for cofirmation-type.rosetta * Remove * Hacked around XSD parser issue * Proper fix --------- Co-authored-by: hugohills-regnosys <[email protected]>
REGnosys · Nov 12, 2024 · d5930e6 · d5930e6
1 parent 8827ec7
commit d5930e6
Show file tree

Hide file tree

Showing 61 changed files with 20,259 additions and 8,844 deletions.
diff --git a/.github/actions/normalize-branch-name/action.yml b/.github/actions/normalize-branch-name/action.yml
@@ -1,12 +1,14 @@
 name: "Normalize branch name"
 description: "Normalizes the branch name"
+
 inputs:
   branch:
     description: "Branch name to normalize. Defaults to $GITHUB_HEAD_REF"
     required: false
 outputs:
   normalized:
     description: "Normalized branch name"
+
 runs:
   using: "node12"
   main: "index.js"

diff --git a/.github/workflows/check-pr.yml b/.github/workflows/check-pr.yml
@@ -1,7 +1,6 @@
 name: PR Checks
 
 on:
-  workflow_dispatch:
   pull_request:
 
 # Cancel previous jobs

diff --git a/.github/workflows/cve-scanning.yml b/.github/workflows/cve-scanning.yml
@@ -1,7 +1,6 @@
 name: CVE Scanning
 
 on:
-  workflow_dispatch:
   push:
     branches:
       - main

diff --git a/.github/workflows/license-scanning.yml b/.github/workflows/license-scanning.yml
@@ -1,7 +1,6 @@
 name: License Scanning for Maven
 
 on:
-  workflow_dispatch:
   push:
     branches:
       - main

diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml
@@ -1,7 +1,6 @@
 name: Build main branch and upload dependency graph
 
 on:
-  workflow_dispatch:
   push:
     branches: [ main ]
 

diff --git a/...ng/src/main/java/com/regnosys/rosetta/generator/java/expression/ExpressionGenerator.xtend b/...ng/src/main/java/com/regnosys/rosetta/generator/java/expression/ExpressionGenerator.xtend
@@ -1059,7 +1059,7 @@ class ExpressionGenerator extends RosettaExpressionSwitch<JavaStatementBuilder,
 
 	override protected caseThenOperation(ThenOperation expr, Context context) {
 		val thenArgCode = expr.argument.javaCode(expr.argument.isMulti ? MAPPER_C.wrapExtends(expr.argument) as JavaType : MAPPER_S.wrapExtends(expr.argument), context.scope)
-		val thenAsVarCode = thenArgCode.declareAsVariable(true, "thenResult", context.scope)
+		val thenAsVarCode = thenArgCode.declareAsVariable(true, "thenArg", context.scope)
 		if (expr.function.parameters.size == 0) {
 			context.scope.createKeySynonym(expr.function.implicitVarInContext, thenArgCode)
 		} else {

diff --git a/...ng/src/main/java/com/regnosys/rosetta/generator/java/expression/TypeCoercionService.xtend b/...ng/src/main/java/com/regnosys/rosetta/generator/java/expression/TypeCoercionService.xtend
@@ -246,13 +246,13 @@ class TypeCoercionService {
 		getItemConversion(actualItemType, expectedItemType, scope)
 			.map[itemConversion|
 				if (actual.isList) {
-					[getListItemConversionExpression(it, itemConversion, scope)]
+					[getListItemConversionExpression(it, itemConversion, expectedItemType.toReferenceType, scope)]
 				} else if (actual.isMapperS) {
-					[getMapperSItemConversionExpression(it, expectedItemType.toReferenceType, itemConversion, scope)]
+					[getMapperSItemConversionExpression(it, itemConversion, expectedItemType.toReferenceType, scope)]
 				} else if (actual.isMapperC) {
-					[getMapperCItemConversionExpression(it, itemConversion, scope)]
+					[getMapperCItemConversionExpression(it, itemConversion, expectedItemType.toReferenceType, scope)]
 				} else if (actual.isMapperListOfLists) {
-					[getMapperListOfListsItemConversionExpression(it, itemConversion, scope)]
+					[getMapperListOfListsItemConversionExpression(it, itemConversion, expectedItemType.toReferenceType, scope)]
 				} else {
 					throw unexpectedWrapperException(actual)
 				}
@@ -488,51 +488,52 @@ class TypeCoercionService {
 	private def JavaExpression getMapperToItemConversionExpression(JavaExpression expression) {
 		JavaExpression.from('''«expression».get()''', expression.expressionType.itemType)
 	}
-	private def JavaExpression getListItemConversionExpression(JavaExpression expression, Function<JavaExpression, ? extends JavaStatementBuilder> itemConversion, JavaScope scope) {
+	private def JavaExpression getListItemConversionExpression(JavaExpression expression, Function<JavaExpression, ? extends JavaStatementBuilder> itemConversion, JavaReferenceType expectedItemType, JavaScope scope) {
 		val actualItemType = expression.expressionType.itemType
 		val lambdaScope = scope.lambdaScope
 		val lambdaParam = lambdaScope.createUniqueIdentifier(actualItemType.simpleName.toFirstLower)
 		val resultItem = itemConversion.apply(new JavaVariable(lambdaParam, actualItemType))
+		val resultType = LIST.wrap(expectedItemType)
 		JavaExpression.from(
 			'''
 			«expression».stream()
-				.<«resultItem.expressionType»>map(«lambdaParam» -> «resultItem.toLambdaBody»)
+				.<«expectedItemType»>map(«lambdaParam» -> «resultItem.toLambdaBody»)
 				.collect(«Collectors».toList())
 			''',
-			LIST.wrap(resultItem.expressionType)
+			resultType
 		)
 	}
-	private def JavaExpression getMapperSItemConversionExpression(JavaExpression expression, JavaReferenceType expectedType, Function<JavaExpression, ? extends JavaStatementBuilder> itemConversion, JavaScope scope) {
+	private def JavaExpression getMapperSItemConversionExpression(JavaExpression expression, Function<JavaExpression, ? extends JavaStatementBuilder> itemConversion, JavaReferenceType expectedItemType, JavaScope scope) {
 		val actualItemType = expression.expressionType.itemType
 		val lambdaScope = scope.lambdaScope
 		val lambdaParam = lambdaScope.createUniqueIdentifier(actualItemType.simpleName.toFirstLower)
 		val inputToItem = new JavaVariable(lambdaParam, actualItemType)
-		val resultType = MAPPER_S.wrap(expectedType)
-		val resultItemNullSafe = convertNullSafe(inputToItem, itemConversion, expectedType, scope)
+		val resultType = MAPPER_S.wrap(expectedItemType)
+		val resultItemNullSafe = convertNullSafe(inputToItem, itemConversion, expectedItemType, scope)
 		JavaExpression.from(
 			'''«expression».<«resultType.itemType»>map("Type coercion", «lambdaParam» -> «resultItemNullSafe.toLambdaBody»)''',
 			resultType
 		)
 	}
-	private def JavaExpression getMapperCItemConversionExpression(JavaExpression expression, Function<JavaExpression, ? extends JavaStatementBuilder> itemConversion, JavaScope scope) {
+	private def JavaExpression getMapperCItemConversionExpression(JavaExpression expression, Function<JavaExpression, ? extends JavaStatementBuilder> itemConversion, JavaReferenceType expectedItemType, JavaScope scope) {
 		val actualItemType = expression.expressionType.itemType
 		val lambdaScope = scope.lambdaScope
 		val lambdaParam = lambdaScope.createUniqueIdentifier(actualItemType.simpleName.toFirstLower)
 		val resultItem = itemConversion.apply(new JavaVariable(lambdaParam, actualItemType))
-		val resultType = MAPPER_C.wrap(resultItem.expressionType)
+		val resultType = MAPPER_C.wrap(expectedItemType)
 		JavaExpression.from(
-			'''«expression».<«resultType.itemType»>map("Type coercion", «lambdaParam» -> «resultItem.toLambdaBody»)''',
+			'''«expression».<«expectedItemType»>map("Type coercion", «lambdaParam» -> «resultItem.toLambdaBody»)''',
 			resultType
 		)
 	}
-	private def JavaExpression getMapperListOfListsItemConversionExpression(JavaExpression expression, Function<JavaExpression, ? extends JavaStatementBuilder> itemConversion, JavaScope scope) {
+	private def JavaExpression getMapperListOfListsItemConversionExpression(JavaExpression expression, Function<JavaExpression, ? extends JavaStatementBuilder> itemConversion, JavaReferenceType expectedItemType, JavaScope scope) {
 		val actualItemType = expression.expressionType.itemType
 		val listToListLambdaScope = scope.lambdaScope
 		val mapperCParam = listToListLambdaScope.createUniqueIdentifier("mapperC")
-		val resultMapperC = getMapperCItemConversionExpression(new JavaVariable(mapperCParam, MAPPER_C.wrap(actualItemType)), itemConversion, listToListLambdaScope)
-		val resultType = MAPPER_LIST_OF_LISTS.wrap(resultMapperC.expressionType.itemType)
+		val resultMapperC = getMapperCItemConversionExpression(new JavaVariable(mapperCParam, MAPPER_C.wrap(actualItemType)), itemConversion, expectedItemType, listToListLambdaScope)
+		val resultType = MAPPER_LIST_OF_LISTS.wrap(expectedItemType)
 		JavaExpression.from(
-			'''«expression».<«resultType.itemType»>mapListToList(«mapperCParam» -> «resultMapperC.toLambdaBody»)''',
+			'''«expression».<«expectedItemType»>mapListToList(«mapperCParam» -> «resultMapperC.toLambdaBody»)''',
 			resultType
 		)
 	}

diff --git a/...ang/src/main/java/com/regnosys/rosetta/generator/java/object/ModelObjectBoilerPlate.xtend b/...ang/src/main/java/com/regnosys/rosetta/generator/java/object/ModelObjectBoilerPlate.xtend
@@ -76,7 +76,7 @@ class ModelObjectBoilerPlate {
 		'''
 			«IF !rMetaAnnotatedType.hasMeta && rMetaAnnotatedType.RType instanceof REnumType»
 				«IF attr.isMulti»
-					_result = 31 * _result + («id» != null ? «id».stream().map(Object::getClass).map(Class::getName).mapToInt(String::hashCode).sum() : 0);
+					_result = 31 * _result + («id» != null ? «id».stream().map(«Object»::getClass).map(«Class»::getName).mapToInt(«String»::hashCode).sum() : 0);
 				«ELSE»
 					_result = 31 * _result + («id» != null ? «id».getClass().getName().hashCode() : 0);
 				«ENDIF»
@@ -105,7 +105,7 @@ class ModelObjectBoilerPlate {
 		val methodScope = scope.methodScope("toString")
 		'''
 		@Override
-		public String toString() {
+		public «String» toString() {
 			return "«classNameFunc.apply(t.name)» {" +
 				«FOR attribute : t.javaAttributes SEPARATOR ' ", " +'»
 					"«attribute.name»=" + this.«methodScope.getIdentifierOrThrow(attribute)» +
@@ -119,9 +119,9 @@ class ModelObjectBoilerPlate {
 		val methodScope = scope.methodScope("equals")
 		'''
 		@Override
-		public boolean equals(Object o) {
+		public boolean equals(«Object» o) {
 			if (this == o) return true;
-			if (o == null || !(o instanceof «RosettaModelObject») || !getType().equals(((RosettaModelObject)o).getType())) return false;
+			if (o == null || !(o instanceof «RosettaModelObject») || !getType().equals(((«RosettaModelObject»)o).getType())) return false;
 			«IF c.hasSuperDataType»
 				if (!super.equals(o)) return false;
 			«ENDIF»

diff --git a/...rc/main/java/com/regnosys/rosetta/generator/java/object/ModelObjectBuilderGenerator.xtend b/...rc/main/java/com/regnosys/rosetta/generator/java/object/ModelObjectBuilderGenerator.xtend
@@ -140,6 +140,7 @@ class ModelObjectBuilderGenerator {
 					}
 
 				«ELSE»
+					@Override
 					public «attribute.toBuilderTypeSingle» getOrCreate«attribute.name.toFirstUpper»(int _index) {
 
 						if («scope.getIdentifierOrThrow(attribute)»==null) {
@@ -173,6 +174,7 @@ class ModelObjectBuilderGenerator {
 		'''
 		«IF attribute.isMulti»
 			@Override
+			@«RosettaAttribute»("«attribute.javaAnnotation»")
 			public «thisName» add«attribute.name.toFirstUpper»(«attribute.toMetaItemJavaType» «scope.getIdentifierOrThrow(attribute)») {
 				if («scope.getIdentifierOrThrow(attribute)»!=null) this.«scope.getIdentifierOrThrow(attribute)».add(«attribute.toBuilder(scope)»);
 				return this;
@@ -208,7 +210,6 @@ class ModelObjectBuilderGenerator {
 			}
 
 			@Override 
-			@«RosettaAttribute»("«attribute.javaAnnotation»")
 			public «thisName» set«attribute.name.toFirstUpper»(«attribute.toMetaJavaType» «scope.getIdentifierOrThrow(attribute)»s) {
 				if («scope.getIdentifierOrThrow(attribute)»s == null)  {
 					this.«scope.getIdentifierOrThrow(attribute)» = new «ArrayList»<>();

diff --git a/rosetta-lang/src/main/java/com/regnosys/rosetta/validation/RosettaSimpleValidator.xtend b/rosetta-lang/src/main/java/com/regnosys/rosetta/validation/RosettaSimpleValidator.xtend
@@ -124,6 +124,8 @@ import com.regnosys.rosetta.rosetta.RosettaExternalFunction
 import com.regnosys.rosetta.types.RChoiceType
 import com.regnosys.rosetta.interpreter.RosettaInterpreter
 import com.google.common.collect.Lists
+import java.util.Collection
+import org.eclipse.xtext.resource.IResourceDescriptions
 import com.regnosys.rosetta.types.RMetaAnnotatedType
 import com.regnosys.rosetta.rosetta.RosettaMetaType
 
@@ -574,7 +576,7 @@ class RosettaSimpleValidator extends AbstractDeclarativeRosettaValidator {
 
 	@Check
 	def void checkClassNameStartsWithCapital(Data classe) {
-		if (!Character.isUpperCase(classe.name.charAt(0))) {
+		if (Character.isLowerCase(classe.name.charAt(0))) {
 			warning("Type name should start with a capital", ROSETTA_NAMED__NAME, INVALID_CASE)
 		}
 	}
@@ -583,7 +585,7 @@ class RosettaSimpleValidator extends AbstractDeclarativeRosettaValidator {
 	def void checkConditionName(Condition condition) {
 		if (condition.name === null && !condition.isConstraintCondition) {
 			warning("Condition name should be specified", ROSETTA_NAMED__NAME, INVALID_NAME)
-		} else if (condition.name !== null && !Character.isUpperCase(condition.name.charAt(0))) {
+		} else if (condition.name !== null && Character.isLowerCase(condition.name.charAt(0))) {
 			warning("Condition name should start with a capital", ROSETTA_NAMED__NAME, INVALID_CASE)
 		}
 	}
@@ -598,14 +600,14 @@ class RosettaSimpleValidator extends AbstractDeclarativeRosettaValidator {
 
 	@Check
 	def void checkFunctionNameStartsWithCapital(Function enumeration) {
-		if (!Character.isUpperCase(enumeration.name.charAt(0))) {
+		if (Character.isLowerCase(enumeration.name.charAt(0))) {
 			warning("Function name should start with a capital", ROSETTA_NAMED__NAME, INVALID_CASE)
 		}
 	}
 
 	@Check
 	def void checkEnumerationNameStartsWithCapital(RosettaEnumeration enumeration) {
-		if (!Character.isUpperCase(enumeration.name.charAt(0))) {
+		if (Character.isLowerCase(enumeration.name.charAt(0))) {
 			warning("Enumeration name should start with a capital", ROSETTA_NAMED__NAME, INVALID_CASE)
 		}
 	}
@@ -614,7 +616,7 @@ class RosettaSimpleValidator extends AbstractDeclarativeRosettaValidator {
 	def void checkAttributeNameStartsWithLowerCase(Attribute attribute) {
 		val annotationAttribute = attribute.eContainer instanceof Annotation
 		val choiceOption = attribute instanceof ChoiceOption
-		if (!choiceOption && !annotationAttribute && !Character.isLowerCase(attribute.name.charAt(0))) {
+		if (!choiceOption && !annotationAttribute && Character.isUpperCase(attribute.name.charAt(0))) {
 			warning("Attribute name should start with a lower case", ROSETTA_NAMED__NAME, INVALID_CASE)
 		}
 	}
@@ -721,27 +723,40 @@ class RosettaSimpleValidator extends AbstractDeclarativeRosettaValidator {
 	// TODO This probably should be made namespace aware
 	@Check(FAST) // switch to NORMAL if it becomes slow
 	def checkTypeNamesAreUnique(RosettaModel model) {
-		val name2attr = HashMultimap.create
-		model.elements.filter(RosettaNamed).filter[!(it instanceof FunctionDispatch)].forEach [ // TODO better FunctionDispatch handling
-			name2attr.put(name, it)
+		// NOTE: this check is done case-insensitive!
+		// This is to prevent clashes in generated code (e.g., MyFoo.java and Myfoo.java),
+		// which case-insensitive file systems such as Mac and Windows do not handle well.
+		val namedRootElems = model.elements.filter(RosettaNamed).filter[!(it instanceof FunctionDispatch)]
+		val descr = getResourceDescriptions(model.eResource)
+
+		val nonAnnotations = namedRootElems.filter[!(it instanceof Annotation)]
+		checkNamesInGroupAreUnique(nonAnnotations.toList, true, descr)
+
+		val annotations = namedRootElems.filter[it instanceof Annotation]
+		checkNamesInGroupAreUnique(annotations.toList, true, descr)
+	}
+	private def checkNamesInGroupAreUnique(Collection<RosettaNamed> namedElems, boolean ignoreCase, IResourceDescriptions descr) {
+		val groupedElems = HashMultimap.create
+		namedElems.filter[name !== null].forEach [
+			groupedElems.put(ignoreCase ? name.toUpperCase : name, it)
 		]
-		val resourceDescription = getResourceDescriptions(model.eResource)
-		for (name : name2attr.keySet) {
-			val valuesByName = name2attr.get(name)
-			if (valuesByName.size > 1) {
-				valuesByName.forEach [
-					if (it.name !== null)
-						error('''Duplicate element named '«name»'«»''', it, ROSETTA_NAMED__NAME, DUPLICATE_ELEMENT_NAME)
+		for (key : groupedElems.keySet) {
+			val group = groupedElems.get(key)
+			if (group.size > 1) {
+				group.forEach [
+					error('''Duplicate element named '«name»'«»''', it, ROSETTA_NAMED__NAME, DUPLICATE_ELEMENT_NAME)
 				]
-			} else if (valuesByName.size == 1 && model.eResource.URI.isPlatformResource) {
-				val EObject toCheck = valuesByName.get(0)
-				val qName = toCheck.fullyQualifiedName
-				val sameNamed = resourceDescription.getExportedObjects(toCheck.eClass(), qName, false).filter [
-					confExtensions.isProjectLocal(model.eResource.URI, it.EObjectURI) && getEClass() !== FUNCTION_DISPATCH
-				].map[EObjectURI]
-				if (sameNamed.size > 1) {
-					error('''Duplicate element named '«qName»' in «sameNamed.filter[toCheck.URI != it].join(', ',[it.lastSegment])»''',
-						toCheck, ROSETTA_NAMED__NAME, DUPLICATE_ELEMENT_NAME)
+			} else if (group.size == 1 && group.head.eResource.URI.isPlatformResource) {
+				val toCheck = group.head
+				if (toCheck instanceof RosettaRootElement) {
+					val qName = toCheck.fullyQualifiedName
+					val sameNamed = descr.getExportedObjects(toCheck.eClass(), qName, false).filter [
+						confExtensions.isProjectLocal(toCheck.eResource.URI, it.EObjectURI) && getEClass() !== FUNCTION_DISPATCH
+					].map[EObjectURI]
+					if (sameNamed.size > 1) {
+						error('''Duplicate element named '«qName»' in «sameNamed.filter[toCheck.URI != it].join(', ',[it.lastSegment])»''',
+							toCheck, ROSETTA_NAMED__NAME, DUPLICATE_ELEMENT_NAME)
+					}
 				}
 			}
 		}

diff --git a/rosetta-runtime/pom.xml b/rosetta-runtime/pom.xml
@@ -55,6 +55,10 @@
 			<groupId>com.fasterxml.jackson.core</groupId>
 			<artifactId>jackson-databind</artifactId>
 		</dependency>
+		<dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jdk8</artifactId>
+        </dependency>
 
 		<!-- Test dependencies -->
 		<dependency>

diff --git a/rosetta-runtime/src/main/java/com/rosetta/util/serialisation/AttributeXMLRepresentation.java b/rosetta-runtime/src/main/java/com/rosetta/util/serialisation/AttributeXMLRepresentation.java
@@ -19,5 +19,6 @@
 public enum AttributeXMLRepresentation {
 	ELEMENT,
 	ATTRIBUTE,
-	VALUE;
+	VALUE,
+	VIRTUAL;
 }
diff --git a/rosetta-runtime/src/main/java/com/rosetta/util/serialisation/RosettaXMLConfiguration.java b/rosetta-runtime/src/main/java/com/rosetta/util/serialisation/RosettaXMLConfiguration.java
@@ -16,16 +16,26 @@
 
 package com.rosetta.util.serialisation;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.stream.Collectors;
 
 import com.fasterxml.jackson.annotation.JsonAnyGetter;
 import com.fasterxml.jackson.annotation.JsonAnySetter;
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.exc.StreamReadException;
+import com.fasterxml.jackson.databind.DatabindException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
 import com.rosetta.model.lib.ModelSymbolId;
 
 public class RosettaXMLConfiguration {
@@ -35,10 +45,24 @@ public class RosettaXMLConfiguration {
 	public RosettaXMLConfiguration(Map<ModelSymbolId, TypeXMLConfiguration> typeConfigMap) {
 		this.typeConfigMap = new TreeMap<>(typeConfigMap);
 	}
+
+	public static RosettaXMLConfiguration load(InputStream input) throws IOException {
+		ObjectMapper xmlConfigurationMapper = new ObjectMapper()
+                .registerModule(new Jdk8Module()) // because RosettaXMLConfiguration contains `Optional` types.
+                .setSerializationInclusion(JsonInclude.Include.NON_ABSENT); // because we want to interpret an absent value as `Optional.empty()`.
+        return xmlConfigurationMapper.readValue(input, RosettaXMLConfiguration.class);
+	}
 
 	public Optional<TypeXMLConfiguration> getConfigurationForType(ModelSymbolId symbolId) {
 		return Optional.ofNullable(typeConfigMap.get(symbolId));
 	}
+
+	public List<ModelSymbolId> getSubstitutionsForType(ModelSymbolId symbolId) {
+		return typeConfigMap.entrySet().stream()
+			.filter(e -> e.getValue().getSubstitutionFor().map(t -> t.equals(symbolId)).orElse(false))
+			.map(e -> e.getKey())
+			.collect(Collectors.toList());
+	}
 
 	@Override
 	public int hashCode() {