Skip to content

Commit

Permalink
Fix finos#786 (finos#853)
Browse files Browse the repository at this point in the history
  • Loading branch information
@SimonCockx
SimonCockx authored Oct 7, 2024
1 parent 842f63f commit 5e14afd
Show file tree
Hide file tree
Showing 2 changed files with 168 additions and 59 deletions.
83 changes: 51 additions & 32 deletions rosetta-ide/rosetta.tmLanguage.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ variables:
namespaceEnd: (?={{identifiersConflictingWithNamespace}}|{{ambiguousRootStart}}|{{unambiguousRootStart}})
rootEnd: (?={{ambiguousRootStart}}|{{unambiguousRootStart}})
unambiguousRootEnd: (?={{unambiguousRootStart}})
sectionStart: '{{wordStart}}((\Qpost-\E)?condition|set|add|inputs|output|alias){{wordEnd}}'
sectionStart: '{{wordStart}}((post-)?condition|set|add|inputs|output|alias){{wordEnd}}'
sectionEnd: (?={{sectionStart}}|{{rootStart}})(?!\bcondition\b)|(?=\bcondition\b\s*({{identifier}})?:)
functionalOperation: '{{wordStart}}(reduce|filter|map|extract|sort|min|max){{wordEnd}}'
listOperationWord: '{{functionalOperation}}|{{wordStart}}(single|multiple|exists|is|absent|only-element|count|flatten|distinct|reverse|first|last|sum){{wordEnd}}'
Expand Down Expand Up @@ -1038,9 +1038,25 @@ repository:
- include: '#documentationFollowedByExpression'

expression:
include: '#parameterizedExpression'
arguments:
extraEnd: ''

expressionWithoutThenOperation:
include: '#parameterizedExpression'
arguments:
extraEnd: '(?={{wordStart}}then{{wordEnd}})|'

expressionWithoutThenAndDefaultOperation:
include: '#parameterizedExpression'
arguments:
extraEnd: '(?={{wordStart}}then|default{{wordEnd}})|'

parameterizedExpression:
parameters: ['extraEnd']
name: meta.expression.rosetta
begin: (?!,|\s|^|$)
end: '{{expressionEnd}}'
end: '{{extraEnd}}{{expressionEnd}}'
patterns:
- include: '#comment'
- name: meta.parens.rosetta
Expand All @@ -1064,12 +1080,28 @@ repository:
- include: '#expression'
- include: '#comma'
- include: '#constructorExpression'
- include: '#functionalOperation'
- name: meta.functional-operation.rosetta
begin: '{{functionalOperation}}'
beginCaptures:
0: { name: keyword.operator.word.rosetta }
end: '{{functionalOperationEnd}}'
patterns:
- include: '#comment'
- begin: (\[)
beginCaptures:
1: { name: punctuation.definition.inline-function.begin.rosetta }
end: (\])
endCaptures:
1: { name: punctuation.definition.inline-function.end.rosetta }
patterns:
- include: '#comment'
- include: '#expression'
- include: '#{{this}}'
- name: meta.then-operation.rosetta
begin: '{{wordStart}}then{{wordEnd}}'
beginCaptures:
0: { name: keyword.operator.word.rosetta }
end: (?=\])|(?={{wordStart}}(then){{wordEnd}})|{{expressionEnd}}
end: (?=\])|(?={{wordStart}}(then){{wordEnd}})|{{extraEnd}}{{expressionEnd}}
patterns:
- include: '#comment'
- begin: \[
Expand Down Expand Up @@ -1105,21 +1137,20 @@ repository:
begin: '{{wordStart}}if{{wordEnd}}'
beginCaptures:
0: { name: keyword.control.conditional.if.rosetta }
end: '{{wordStart}}(then){{wordEnd}}|{{expressionEnd}}'
end: '{{wordStart}}(then){{wordEnd}}|{{extraEnd}}{{expressionEnd}}'
endCaptures:
1: { name: keyword.control.conditional.then.rosetta }
patterns:
- include: '#comment'
- include: '#expression'
- include: '#expressionWithoutThenOperation'
- name: keyword.control.conditional.else.rosetta
match: '{{wordStart}}else{{wordEnd}}'
- begin: '{{wordStart}}(optional|required){{wordEnd}}'
- begin: '{{wordStart}}(optional|required){{wordEnd}}\s+({{wordStart}}choice{{wordEnd}})?'
beginCaptures:
0: { name: keyword.operator.word.rosetta }
end: '{{expressionEndIgnoringComma}}'
1: { name: keyword.operator.word.rosetta }
2: { name: keyword.operator.word.rosetta }
end: '{{extraEnd}}{{expressionEndIgnoringComma}}'
patterns:
- name: keyword.operator.word.rosetta
match: '{{wordStart}}choice{{wordEnd}}'
- name: variable.other.member.rosetta
match: '{{identifier}}'
- name: punctuation.separator.member.rosetta
Expand All @@ -1137,7 +1168,7 @@ repository:
begin: '{{wordStart}}switch{{wordEnd}}'
beginCaptures:
0: { name: keyword.operator.word.rosetta }
end: '{{expressionEndIgnoringComma}}'
end: '{{extraEnd}}{{expressionEndIgnoringComma}}'
patterns:
- include: '#comment'
- begin: '{{wordStart}}then{{wordEnd}}'
Expand All @@ -1147,7 +1178,14 @@ repository:
patterns:
- include: '#comment'
- include: '#expression'
- include: '#expression'
- begin: '{{wordStart}}default{{wordEnd}}'
beginCaptures:
0: { name: keyword.control.conditional.default.rosetta }
end: '{{expressionEnd}}'
patterns:
- include: '#comment'
- include: '#expression'
- include: '#expressionWithoutThenAndDefaultOperation'
- name: keyword.operator.word.rosetta
match: '{{listOperationWord}}|{{wordStart}}(any|all|or|and|contains|default|disjoint|join|only|exists|as-key|one-of|as(?!-)|to-number|to-int|to-time|to-date|to-date-time|to-zoned-date-time|to-string){{wordEnd}}'
- name: keyword.operator.rosetta
Expand Down Expand Up @@ -1185,25 +1223,6 @@ repository:
- name: constant.language.rosetta
match: '{{wordStart}}(True|False|empty){{wordEnd}}'

functionalOperation:
name: meta.functional-operation.rosetta
begin: '{{functionalOperation}}'
beginCaptures:
0: { name: keyword.operator.word.rosetta }
end: '{{functionalOperationEnd}}'
patterns:
- include: '#comment'
- begin: (\[)
beginCaptures:
1: { name: punctuation.definition.inline-function.begin.rosetta }
end: (\])
endCaptures:
1: { name: punctuation.definition.inline-function.end.rosetta }
patterns:
- include: '#comment'
- include: '#expression'
- include: '#expression'

documentationAndAnnotationsFollowedByExpression: # Special rule to circumvent clashing of '[' and '<' in expressions
name: meta.documentation-and-annotations-followed-by-expression.rosetta
begin: (?!\s|^|$)
Expand Down
144 changes: 117 additions & 27 deletions rosetta-ide/src/main/java/com/regnosys/rosetta/ide/textmate/GenerateTmGrammar.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,14 @@
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.Map.Entry;
import java.util.function.Function;
import java.util.function.Predicate;
Expand All @@ -49,7 +53,7 @@
import com.regnosys.rosetta.services.RosettaGrammarAccess;

public class GenerateTmGrammar {
private static List<String> ignoredRosettaKeywords = List.of("..", "namespace", "condition", /* @Compat */"qualifiedType", "calculationType");
private static List<String> ignoredRosettaKeywords = List.of("..", "namespace", "condition", "required", "optional", /* @Compat */"qualifiedType", "calculationType");

/**
* param 0: path to input yaml file
Expand All @@ -76,11 +80,86 @@ private void generateTmLanguage(String inputPath, String outputPath) throws IOEx
Map<String, String> variables = readVariables(input);
input.remove("variables");
applyVariablesRecursively(input, variables);
validateTm(input);
inlineParameterizedIncludes(input);
writeJson(input, outputPath);
validateTm(input);
}

private void inlineParameterizedIncludes(Map<Object, Object> input) {
Map<String, Set<Map<String, String>>> argumentMapPerInclude = new HashMap<>();
gatherIncludeArguments(input, argumentMapPerInclude);
inlineParameterizedIncludesRecursively(input, argumentMapPerInclude);
}

@SuppressWarnings("unchecked")
private void gatherIncludeArguments(Object input, Map<String, Set<Map<String, String>>> argumentMapPerInclude) {
if (input instanceof Map) {
Map<Object, Object> inputMap = (Map<Object, Object>)input;
Object rawInclude = inputMap.get("include");
if (rawInclude != null && rawInclude instanceof String && inputMap.containsKey("arguments")) {
Map<String, String> argumentMap = readStringMap(inputMap.get("arguments"), "argument");
String include = ((String)rawInclude).substring(1);
argumentMapPerInclude.computeIfAbsent(include, a -> new LinkedHashSet<>()).add(argumentMap);

String inlineName = toInlineName(include, argumentMap);
inputMap.put("include", "#" + inlineName);
inputMap.remove("arguments");
}
for (Entry<?, ?> node : inputMap.entrySet()) {
gatherIncludeArguments(node.getValue(), argumentMapPerInclude);
}
} else if (input instanceof List) {
for (Object item : (List<?>)input) {
gatherIncludeArguments(item, argumentMapPerInclude);
}
}
}

@SuppressWarnings("unchecked")
private void inlineParameterizedIncludesRecursively(Object input, Map<String, Set<Map<String, String>>> argumentMapPerInclude) {
if (input instanceof Map) {
Map<?, ?> inputMap = (Map<?, ?>)input;
Object rawRepo = inputMap.get("repository");
if (rawRepo != null && rawRepo instanceof Map<?, ?>) {
Map<Object, Object> repo = (Map<Object, Object>)rawRepo;
Gson gson = new GsonBuilder().disableHtmlEscaping().create();
Map<Object, Object> inlinedDefinitions = new LinkedHashMap<>();
for (Entry<Object, Object> definitionEntry : repo.entrySet()) {
Object definitionName = definitionEntry.getKey();
Object rawDefinition = definitionEntry.getValue();
if (rawDefinition instanceof Map<?, ?>) {
Map<?, ?> definition = (Map<?, ?>) rawDefinition;
if (definition.containsKey("parameters")) {
for (Map<String, String> argumentMap : argumentMapPerInclude.getOrDefault(definitionName, Collections.emptySet())) {
String inlineName = toInlineName((String)definitionName, argumentMap);

Map<?, ?> inlinedDefinition = gson.fromJson(gson.toJson(definition), Map.class);
inlinedDefinition.remove("parameters");
argumentMap.put("this", inlineName);
applyVariablesRecursively(inlinedDefinition, argumentMap);
inlinedDefinitions.put(inlineName, inlinedDefinition);
}
}
}
}
repo.putAll(inlinedDefinitions);
repo.entrySet().removeIf(e -> e.getValue() instanceof Map<?, ?> && ((Map<?, ?>)e.getValue()).containsKey("parameters"));
}
for (Entry<?, ?> node : inputMap.entrySet()) {
inlineParameterizedIncludesRecursively(node.getValue(), argumentMapPerInclude);
}
} else if (input instanceof List) {
for (Object item : (List<?>)input) {
inlineParameterizedIncludesRecursively(item, argumentMapPerInclude);
}
}
}
private String toInlineName(String include, Map<String, String> arguments) {
return include + new TreeMap<>(arguments).entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(",", "(", ")"));
}

private void validateTm(Map<Object, Object> input) throws ConfigurationException {
ensureNoUnknownVariables(input);
Map<Object, Object> namedPatterns = findNamedPatterns(input);
List<TmValue<Object>> allPatterns = findAllPatterns(input, new ArrayList<>());
for (TmValue<Object> pattern: allPatterns) {
Expand All @@ -90,6 +169,27 @@ private void validateTm(Map<Object, Object> input) throws ConfigurationException
ensureAllRosettaKeywordsAreSupported(input);
}

private void ensureNoUnknownVariables(Object input) throws ConfigurationException {
if (input instanceof Map) {
Map<?, ?> inputMap = (Map<?, ?>)input;
for (Entry<?, ?> node : inputMap.entrySet()) {
if (node.getValue() instanceof String) {
Matcher unknownVariableMatcher = variablePattern.matcher((String)node.getValue());
List<MatchResult> unknownVariables = unknownVariableMatcher.results().collect(Collectors.toList());
if (unknownVariables.size() > 0) {
throw new ConfigurationException("At " + node.getKey() + ": Unknown variable(s): " + unknownVariables.stream().map(v -> v.group(1)).collect(Collectors.joining(", ")));
}
} else {
ensureNoUnknownVariables(node.getValue());
}
}
} else if (input instanceof List) {
for (Object item : (List<?>)input) {
ensureNoUnknownVariables(item);
}
}
}

private void ensureAllRosettaKeywordsAreSupported(Map<Object, Object> input) throws ConfigurationException {
List<Pattern> regexes = findAllRegexes(input);

Expand Down Expand Up @@ -297,36 +397,29 @@ private void writeJson(Map<Object, Object> input, String outputPath) throws IOEx
}
}

private Map<String, String> readVariables(Map<Object, Object> yaml) throws ConfigurationException {
LinkedHashMap<String, String> result = new LinkedHashMap<>(yaml.size());

private Map<String, String> readVariables(Map<Object, Object> yaml) throws ConfigurationException {
Object rawVariables = yaml.get("variables");
if (!(rawVariables instanceof Map)) {
return result;
return readStringMap(rawVariables, "variable");
}
private Map<String, String> readStringMap(Object raw, String errorVarName) {
if (!(raw instanceof Map)) {
return Collections.emptyMap();
}
Map<?, ?> variables = (Map<?, ?>)rawVariables;
Map<?, ?> variables = (Map<?, ?>)raw;
LinkedHashMap<String, String> result = new LinkedHashMap<>(variables.size());
for (Entry<?, ?> variable : variables.entrySet()) {
if (variable.getValue() instanceof String) {
String rawValue = (String)variable.getValue();
try {
result.put((String)variable.getKey(), applyVariables(rawValue, result));
} catch (ConfigurationException e) {
throw new ConfigurationException("While parsing variable `" + variable.getKey() + "`: " + e.getExplanation());
}
result.put((String)variable.getKey(), applyVariables(rawValue, result));
}
}
return result;
}

private String applyVariables(String input, Map<String, String> variables) throws ConfigurationException {
private String applyVariables(String input, Map<String, String> variables) {
for (Entry<String, String> variable : variables.entrySet()) {
input = applyVariable(input, variable);
}
Matcher unknownVariableMatcher = variablePattern.matcher(input);
List<MatchResult> unknownVariables = unknownVariableMatcher.results().collect(Collectors.toList());
if (unknownVariables.size() > 0) {
throw new ConfigurationException("Unknown variable(s): " + unknownVariables.stream().map(v -> v.group(1)).collect(Collectors.joining(", ")));
}
return input;
}

Expand All @@ -335,15 +428,12 @@ private String applyVariable(String input, Entry<String, String> variable) {
}

@SuppressWarnings("unchecked")
private void applyVariablesRecursively(Object input, Map<String, String> variables) throws ConfigurationException {
private void applyVariablesRecursively(Object input, Map<String, String> variables) {
if (input instanceof Map) {
for (Entry<?, ?> node : ((Map<?, ?>)input).entrySet()) {
if (node.getValue() instanceof String) {
try {
((Entry<?, String>)node).setValue(applyVariables((String)node.getValue(), variables));
} catch (ConfigurationException e) {
throw new ConfigurationException("At " + node.getKey() + ": " + e.getExplanation());
}
Map<?, ?> inputMap = (Map<?, ?>)input;
for (Entry<?, ?> node : inputMap.entrySet()) {
if (node.getValue() instanceof String) {
((Entry<?, String>)node).setValue(applyVariables((String)node.getValue(), variables));
} else {
applyVariablesRecursively(node.getValue(), variables);
}
Expand Down

0 comments on commit 5e14afd

Please sign in to comment.