From cef4c2aac8a5cfcd40d0ff1e57d2fd69738140c4 Mon Sep 17 00:00:00 2001 From: Tae Hyung Kim Date: Sat, 5 Nov 2022 10:26:46 -0700 Subject: [PATCH] ICU Message Syntax Parser (#112390) * init * code generation * improve syntax error, add tests * add tests and fix bugs * code generation fix * fix all tests :) * fix bug * init * fix all code gen issues * FIXED ALL TESTS :D * add license * remove trailing spaces * remove print * tests fix * specify type annotation * fix test * lint * fix todos * fix subclass issues * final fix; flutter gallery runs * escaping for later pr * fix comment * address PR comments * more * more descriptive errors * last fixes --- .../lib/src/localizations/gen_l10n.dart | 542 ++++++++--------- .../src/localizations/gen_l10n_templates.dart | 81 +-- .../lib/src/localizations/gen_l10n_types.dart | 83 ++- .../localizations/localizations_utils.dart | 29 +- .../lib/src/localizations/message_parser.dart | 543 ++++++++++++++++++ .../generate_localizations_test.dart | 175 ++++-- .../general.shard/message_parser_test.dart | 491 ++++++++++++++++ .../test/integration.shard/gen_l10n_test.dart | 79 +-- .../test_data/gen_l10n_project.dart | 20 +- 9 files changed, 1590 insertions(+), 453 deletions(-) create mode 100644 packages/flutter_tools/lib/src/localizations/message_parser.dart create mode 100644 packages/flutter_tools/test/general.shard/message_parser_test.dart diff --git a/packages/flutter_tools/lib/src/localizations/gen_l10n.dart b/packages/flutter_tools/lib/src/localizations/gen_l10n.dart index e14fbdd17b9e..81921b7b9cc6 100644 --- a/packages/flutter_tools/lib/src/localizations/gen_l10n.dart +++ b/packages/flutter_tools/lib/src/localizations/gen_l10n.dart @@ -13,6 +13,7 @@ import '../flutter_manifest.dart'; import 'gen_l10n_templates.dart'; import 'gen_l10n_types.dart'; import 'localizations_utils.dart'; +import 'message_parser.dart'; /// Run the localizations generation script with the configuration [options]. LocalizationsGenerator generateLocalizations({ @@ -84,22 +85,30 @@ String _defaultSyntheticPackagePath(FileSystem fileSystem) => fileSystem.path.jo /// localizations tool. String _syntheticL10nPackagePath(FileSystem fileSystem) => fileSystem.path.join(_defaultSyntheticPackagePath(fileSystem), 'gen_l10n'); +// Generate method parameters and also infer the correct types from the usage of the placeholders +// For example, if placeholders are used for plurals and no type was specified, then the type will +// automatically set to 'num'. Similarly, if such placeholders are used for selects, then the type +// will be set to 'String'. For such placeholders that are used for both, we should throw an error. +// TODO(thkim1011): Let's store the output of this function in the Message class, so that we don't +// recompute this. See https://github.com/flutter/flutter/issues/112709 List generateMethodParameters(Message message) { - assert(message.placeholders.isNotEmpty); - final Placeholder? countPlaceholder = message.isPlural ? message.getCountPlaceholder() : null; return message.placeholders.map((Placeholder placeholder) { - final String? type = placeholder == countPlaceholder ? 'num' : placeholder.type; - return '$type ${placeholder.name}'; + return '${placeholder.type} ${placeholder.name}'; }).toList(); } +// Similar to above, but is used for passing arguments into helper functions. +List generateMethodArguments(Message message) { + return message.placeholders.map((Placeholder placeholder) => placeholder.name).toList(); +} + String generateDateFormattingLogic(Message message) { if (message.placeholders.isEmpty || !message.placeholdersRequireFormatting) { return '@(none)'; } final Iterable formatStatements = message.placeholders - .where((Placeholder placeholder) => placeholder.isDate) + .where((Placeholder placeholder) => placeholder.requiresDateFormatting) .map((Placeholder placeholder) { final String? placeholderFormat = placeholder.format; if (placeholderFormat == null) { @@ -130,7 +139,7 @@ String generateDateFormattingLogic(Message message) { } return dateFormatCustomTemplate .replaceAll('@(placeholder)', placeholder.name) - .replaceAll('@(format)', generateString(placeholderFormat)); + .replaceAll('@(format)', "'${generateString(placeholderFormat)}'"); }); return formatStatements.isEmpty ? '@(none)' : formatStatements.join(); @@ -142,7 +151,7 @@ String generateNumberFormattingLogic(Message message) { } final Iterable formatStatements = message.placeholders - .where((Placeholder placeholder) => placeholder.isNumber) + .where((Placeholder placeholder) => placeholder.requiresNumFormatting) .map((Placeholder placeholder) { final String? placeholderFormat = placeholder.format; if (!placeholder.hasValidNumberFormat || placeholderFormat == null) { @@ -158,7 +167,7 @@ String generateNumberFormattingLogic(Message message) { if (parameter.value is num) { return '${parameter.name}: ${parameter.value}'; } else { - return '${parameter.name}: ${generateString(parameter.value.toString())}'; + return "${parameter.name}: '${generateString(parameter.value.toString())}'"; } }, ); @@ -178,279 +187,24 @@ String generateNumberFormattingLogic(Message message) { return formatStatements.isEmpty ? '@(none)' : formatStatements.join(); } -/// To make it easier to parse plurals or select messages, temporarily replace -/// each "{placeholder}" parameter with "#placeholder#" for example. -String _replacePlaceholdersBraces( - String translationForMessage, - Iterable placeholders, - String replacementBraces, -) { - assert(replacementBraces.length == 2); - String easyMessage = translationForMessage; - for (final Placeholder placeholder in placeholders) { - easyMessage = easyMessage.replaceAll( - '{${placeholder.name}}', - '${replacementBraces[0]}${placeholder.name}${replacementBraces[1]}', - ); - } - return easyMessage; -} - -/// Replaces message with the interpolated variable name of the given placeholders -/// with the ability to change braces to something other than {...}. -/// -/// Examples: -/// -/// * Replacing `{userName}`. -/// ```dart -/// final message = 'Hello my name is {userName}'; -/// final transformed = _replacePlaceholdersWithVariables(message, placeholders); -/// // transformed == 'Hello my name is $userName' -/// ``` -/// * Replacing `#choice#`. -/// ```dart -/// final message = 'I would like to have some #choice#'; -/// final transformed = _replacePlaceholdersWithVariables(message, placeholders, '##'); -/// transformed == 'I would like to have some $choice' -/// ``` -String _replacePlaceholdersWithVariables(String message, Iterable placeholders, [String braces = '{}']) { - assert(braces.length == 2); - String messageWithValues = message; - for (final Placeholder placeholder in placeholders) { - String variable = placeholder.name; - if (placeholder.requiresFormatting) { - variable += 'String'; - } - messageWithValues = messageWithValues.replaceAll( - '${braces[0]}${placeholder.name}${braces[1]}', - _needsCurlyBracketStringInterpolation(messageWithValues, placeholder.name) - ? '\${$variable}' - : '\$$variable' - ); - } - return messageWithValues; -} - -String _generatePluralMethod(Message message, String translationForMessage) { - if (message.placeholders.isEmpty) { - throw L10nException( - 'Unable to find placeholders for the plural message: ${message.resourceId}.\n' - 'Check to see if the plural message is in the proper ICU syntax format ' - 'and ensure that placeholders are properly specified.' - ); - } - - final String easyMessage = _replacePlaceholdersBraces(translationForMessage, message.placeholders, '##'); - - final Placeholder countPlaceholder = message.getCountPlaceholder(); - const Map pluralIds = { - '=0': 'zero', - '=1': 'one', - '=2': 'two', - 'few': 'few', - 'many': 'many', - 'other': 'other', - }; - - final List pluralLogicArgs = []; - for (final String pluralKey in pluralIds.keys) { - final RegExp expRE = RegExp('($pluralKey)\\s*{([^}]+)}'); - final RegExpMatch? match = expRE.firstMatch(easyMessage); - if (match != null && match.groupCount == 2) { - final String argValue = _replacePlaceholdersWithVariables(generateString(match.group(2)!), message.placeholders, '##'); - pluralLogicArgs.add(' ${pluralIds[pluralKey]}: $argValue'); - } - } - - final List parameters = message.placeholders.map((Placeholder placeholder) { - final String? placeholderType = placeholder == countPlaceholder ? 'num' : placeholder.type; - return '$placeholderType ${placeholder.name}'; - }).toList(); - - final String comment = message.description ?? 'No description provided in @${message.resourceId}'; - - if (translationForMessage.startsWith('{') && translationForMessage.endsWith('}')) { - return pluralMethodTemplate - .replaceAll('@(comment)', comment) - .replaceAll('@(name)', message.resourceId) - .replaceAll('@(dateFormatting)', generateDateFormattingLogic(message)) - .replaceAll('@(numberFormatting)', generateNumberFormattingLogic(message)) - .replaceAll('@(parameters)', parameters.join(', ')) - .replaceAll('@(count)', countPlaceholder.name) - .replaceAll('@(pluralLogicArgs)', pluralLogicArgs.join(',\n')) - .replaceAll('@(none)\n', ''); - } - - const String variable = 'pluralString'; - final String string = _replaceWithVariable(translationForMessage, variable); - return pluralMethodTemplateInString - .replaceAll('@(comment)', comment) - .replaceAll('@(name)', message.resourceId) - .replaceAll('@(dateFormatting)', generateDateFormattingLogic(message)) - .replaceAll('@(numberFormatting)', generateNumberFormattingLogic(message)) - .replaceAll('@(parameters)', parameters.join(', ')) - .replaceAll('@(variable)', variable) - .replaceAll('@(count)', countPlaceholder.name) - .replaceAll('@(pluralLogicArgs)', pluralLogicArgs.join(',\n')) - .replaceAll('@(none)\n', '') - .replaceAll('@(string)', string); -} - -String _replaceWithVariable(String translation, String variable) { - String prefix = generateString(translation.substring(0, translation.indexOf('{'))); - prefix = prefix.substring(0, prefix.length - 1); - String suffix = generateString(translation.substring(translation.lastIndexOf('}') + 1)); - suffix = suffix.substring(1); - - // escape variable when the suffix can be combined with the variable - if (suffix.isNotEmpty && !suffix.startsWith(' ')) { - variable = '{$variable}'; - } - return prefix + r'$' + variable + suffix; -} - -String _generateSelectMethod(Message message, String translationForMessage) { - if (message.placeholders.isEmpty) { - throw L10nException( - 'Unable to find placeholders for the select message: ${message.resourceId}.\n' - 'Check to see if the select message is in the proper ICU syntax format ' - 'and ensure that placeholders are properly specified.' - ); - } - - final String easyMessage = _replacePlaceholdersBraces(translationForMessage, message.placeholders, '##'); - - final List cases = []; - - final RegExpMatch? selectMatch = LocalizationsGenerator._selectRE.firstMatch(easyMessage); - String? choice; - if (selectMatch != null && selectMatch.groupCount == 2) { - choice = selectMatch.group(1); - final String pattern = selectMatch.group(2)!; - final RegExp patternRE = RegExp(r'\s*([\w\d]+)\s*\{(.*?)\}'); - for (final RegExpMatch patternMatch in patternRE.allMatches(pattern)) { - if (patternMatch.groupCount == 2) { - String value = patternMatch.group(2)! - .replaceAll("'", r"\'") - .replaceAll('"', r'\"'); - value = _replacePlaceholdersWithVariables(value, message.placeholders, '##'); - cases.add( - " '${patternMatch.group(1)}': '$value'", - ); - } - } - } else { - throw L10nException( - 'Incorrect select message format for: ${message.resourceId}.\n' - 'Check to see if the select message is in the proper ICU syntax format.' - ); - } - - final List parameters = message.placeholders.map((Placeholder placeholder) { - final String placeholderType = placeholder.type ?? 'object'; - return '$placeholderType ${placeholder.name}'; - }).toList(); - - final String description = message.description ?? 'No description provided in @${message.resourceId}'; - - if (translationForMessage.startsWith('{') && translationForMessage.endsWith('}')) { - return selectMethodTemplate - .replaceAll('@(name)', message.resourceId) - .replaceAll('@(parameters)', parameters.join(', ')) - .replaceAll('@(choice)', choice!) - .replaceAll('@(cases)', cases.join(',\n').trim()) - .replaceAll('@(description)', description); - } - - const String variable = 'selectString'; - final String string = _replaceWithVariable(translationForMessage, variable); - return selectMethodTemplateInString - .replaceAll('@(name)', message.resourceId) - .replaceAll('@(parameters)', parameters.join(', ')) - .replaceAll('@(variable)', variable) - .replaceAll('@(choice)', choice!) - .replaceAll('@(cases)', cases.join(',\n').trim()) - .replaceAll('@(description)', description) - .replaceAll('@(string)', string); -} - -bool _needsCurlyBracketStringInterpolation(String messageString, String placeholder) { - final int placeholderIndex = messageString.indexOf(placeholder); - // This means that this message does not contain placeholders/parameters, - // since one was not found in the message. - if (placeholderIndex == -1) { - return false; - } - - final bool isPlaceholderEndOfSubstring = placeholderIndex + placeholder.length + 2 == messageString.length; - - if (placeholderIndex > 2 && !isPlaceholderEndOfSubstring) { - // Normal case - // Examples: - // "'The number of {hours} elapsed is: 44'" // no curly brackets. - // "'哈{hours}哈'" // no curly brackets. - // "'m#hours#m'" // curly brackets. - // "'I have to work _#hours#_' sometimes." // curly brackets. - final RegExp commonCaseRE = RegExp('[^a-zA-Z_][#{]$placeholder[#}][^a-zA-Z_]'); - return !commonCaseRE.hasMatch(messageString); - } else if (placeholderIndex == 2) { - // Example: - // "'{hours} elapsed.'" // no curly brackets - // '#placeholder# ' // no curly brackets - // '#placeholder#m' // curly brackets - final RegExp startOfString = RegExp('[#{]$placeholder[#}][^a-zA-Z_]'); - return !startOfString.hasMatch(messageString); - } else { - // Example: - // "'hours elapsed: {hours}'" - // "'Time elapsed: {hours}'" // no curly brackets - // ' #placeholder#' // no curly brackets - // 'm#placeholder#' // curly brackets - final RegExp endOfString = RegExp('[^a-zA-Z_][#{]$placeholder[#}]'); - return !endOfString.hasMatch(messageString); - } -} - -String _generateMethod(Message message, String translationForMessage) { - String generateMessage() { - return _replacePlaceholdersWithVariables(generateString(translationForMessage), message.placeholders); - } - - if (message.isPlural) { - return _generatePluralMethod(message, translationForMessage); - } - - if (message.isSelect) { - return _generateSelectMethod(message, translationForMessage); - } - - if (message.placeholdersRequireFormatting) { - return formatMethodTemplate - .replaceAll('@(name)', message.resourceId) - .replaceAll('@(parameters)', generateMethodParameters(message).join(', ')) - .replaceAll('@(dateFormatting)', generateDateFormattingLogic(message)) - .replaceAll('@(numberFormatting)', generateNumberFormattingLogic(message)) - .replaceAll('@(message)', generateMessage()) - .replaceAll('@(none)\n', ''); - } - - if (message.placeholders.isNotEmpty) { - return methodTemplate - .replaceAll('@(name)', message.resourceId) - .replaceAll('@(parameters)', generateMethodParameters(message).join(', ')) - .replaceAll('@(message)', generateMessage()); - } - - return getterTemplate - .replaceAll('@(name)', message.resourceId) - .replaceAll('@(message)', generateMessage()); -} +/// List of possible cases for plurals defined the ICU messageFormat syntax. +Map pluralCases = { + '0': 'zero', + '1': 'one', + '2': 'two', + 'zero': 'zero', + 'one': 'one', + 'two': 'two', + 'few': 'few', + 'many': 'many', + 'other': 'other', +}; String generateBaseClassMethod(Message message, LocaleInfo? templateArbLocale) { final String comment = message.description ?? 'No description provided for @${message.resourceId}.'; final String templateLocaleTranslationComment = ''' /// In $templateArbLocale, this message translates to: - /// **${generateString(message.value)}**'''; + /// **'${generateString(message.value)}'**'''; if (message.placeholders.isNotEmpty) { return baseClassMethodTemplate @@ -806,6 +560,10 @@ class LocalizationsGenerator { /// ['es', 'en'] is passed in, the 'es' locale will take priority over 'en'. final List preferredSupportedLocales; + // Whether we need to import intl or not. This flag is updated after parsing + // all of the messages. + bool requiresIntlImport = false; + /// The list of all arb path strings in [inputDirectory]. List get arbPathStrings { return _allBundles.bundles.map((AppResourceBundle bundle) => bundle.file.path).toList(); @@ -870,8 +628,6 @@ class LocalizationsGenerator { /// Logger to be used during the execution of the script. Logger logger; - static final RegExp _selectRE = RegExp(r'\{([\w\s,]*),\s*select\s*,\s*([\w\d]+\s*\{.*\})+\s*\}'); - static bool _isNotReadable(FileStat fileStat) { final String rawStatString = fileStat.modeString(); // Removes potential prepended permission bits, such as '(suid)' and '(guid)'. @@ -1087,7 +843,7 @@ class LocalizationsGenerator { // files in inputDirectory. Also initialized: supportedLocales. void loadResources() { _allMessages = _templateBundle.resourceIds.map((String id) => Message( - _templateBundle.resources, id, areResourceAttributesRequired, + _templateBundle.resources, id, areResourceAttributesRequired, )); for (final String resourceId in _templateBundle.resourceIds) { if (!_isValidGetterAndMethodName(resourceId)) { @@ -1148,25 +904,11 @@ class LocalizationsGenerator { return _generateMethod( message, + bundle.file.basename, bundle.translationFor(message) ?? templateBundle.translationFor(message)!, ); }); - for (final Message message in messages) { - if (message.isPlural) { - if (message.placeholders.isEmpty) { - throw L10nException( - 'Unable to find placeholders for the plural message: ${message.resourceId}.\n' - 'Check to see if the plural message is in the proper ICU syntax format ' - 'and ensure that placeholders are properly specified.'); - } - final Placeholder countPlaceholder = message.getCountPlaceholder(); - if (countPlaceholder.type != null && countPlaceholder.type != 'num') { - logger.printWarning("Placeholders for plurals are automatically converted to type 'num' for the message: ${message.resourceId}."); - } - } - } - return classFileTemplate .replaceAll('@(header)', header.isEmpty ? '' : '$header\n\n') .replaceAll('@(language)', describeLocale(locale.toString())) @@ -1175,7 +917,7 @@ class LocalizationsGenerator { .replaceAll('@(class)', '$className${locale.camelCase()}') .replaceAll('@(localeName)', locale.toString()) .replaceAll('@(methods)', methods.join('\n\n')) - .replaceAll('@(requiresIntlImport)', _requiresIntlImport() ? "import 'package:intl/intl.dart' as intl;\n\n" : ''); + .replaceAll('@(requiresIntlImport)', requiresIntlImport ? "import 'package:intl/intl.dart' as intl;\n\n" : ''); } String _generateSubclass( @@ -1194,7 +936,7 @@ class LocalizationsGenerator { final Iterable methods = messages .where((Message message) => bundle.translationFor(message) != null) - .map((Message message) => _generateMethod(message, bundle.translationFor(message)!)); + .map((Message message) => _generateMethod(message, bundle.file.basename, bundle.translationFor(message)!)); return subclassTemplate .replaceAll('@(language)', describeLocale(locale.toString())) @@ -1328,7 +1070,7 @@ class LocalizationsGenerator { .replaceAll('@(messageClassImports)', sortedClassImports.join('\n')) .replaceAll('@(delegateClass)', delegateClass) .replaceAll('@(requiresFoundationImport)', useDeferredLoading ? '' : "import 'package:flutter/foundation.dart';") - .replaceAll('@(requiresIntlImport)', _requiresIntlImport() ? "import 'package:intl/intl.dart' as intl;" : '') + .replaceAll('@(requiresIntlImport)', requiresIntlImport ? "import 'package:intl/intl.dart' as intl;" : '') .replaceAll('@(canBeNullable)', usesNullableGetter ? '?' : '') .replaceAll('@(needsNullCheck)', usesNullableGetter ? '' : '!') // Removes all trailing whitespace from the generated file. @@ -1337,11 +1079,207 @@ class LocalizationsGenerator { .replaceAll('\n\n\n', '\n\n'); } - bool _requiresIntlImport() => _allMessages.any((Message message) { - return message.isPlural - || message.isSelect - || message.placeholdersRequireFormatting; - }); + String _generateMethod(Message message, String filename, String translationForMessage) { + // Determine if we must import intl for date or number formatting. + if (message.placeholdersRequireFormatting) { + requiresIntlImport = true; + } + + final Node node = Parser(message.resourceId, filename, translationForMessage).parse(); + // If parse tree is only a string, then return a getter method. + if (node.children.every((Node child) => child.type == ST.string)) { + // Use the parsed translation to handle escaping with the same behavior. + return getterTemplate + .replaceAll('@(name)', message.resourceId) + .replaceAll('@(message)', "'${generateString(node.children.map((Node child) => child.value!).join())}'"); + } + + final List helperMethods = []; + + // Get a unique helper method name. + int methodNameCount = 0; + String getHelperMethodName() { + return '_${message.resourceId}${methodNameCount++}'; + } + + // Do a DFS post order traversal, generating dependent + // placeholder, plural, select helper methods, and combine these into + // one message. Returns the method/placeholder to use in parent string. + HelperMethod generateHelperMethods(Node node, { bool isRoot = false }) { + final Set dependentPlaceholders = {}; + switch (node.type) { + case ST.message: + final List helpers = node.children.map((Node node) { + if (node.type == ST.string) { + return HelperMethod({}, string: node.value); + } + final HelperMethod helper = generateHelperMethods(node); + dependentPlaceholders.addAll(helper.dependentPlaceholders); + return helper; + }).toList(); + final String messageString = generateReturnExpr(helpers); + + // If the message is just a normal string, then only return the string. + if (dependentPlaceholders.isEmpty) { + return HelperMethod(dependentPlaceholders, string: messageString); + } + + // For messages, if we are generating the actual overridden method, then we should also deal with + // date and number formatting here. + final String helperMethodName = getHelperMethodName(); + final HelperMethod messageHelper = HelperMethod(dependentPlaceholders, helper: helperMethodName); + if (isRoot) { + helperMethods.add(methodTemplate + .replaceAll('@(name)', message.resourceId) + .replaceAll('@(parameters)', generateMethodParameters(message).join(', ')) + .replaceAll('@(dateFormatting)', generateDateFormattingLogic(message)) + .replaceAll('@(numberFormatting)', generateNumberFormattingLogic(message)) + .replaceAll('@(message)', messageString) + .replaceAll('@(none)\n', '') + ); + } else { + helperMethods.add(messageHelperTemplate + .replaceAll('@(name)', helperMethodName) + .replaceAll('@(parameters)', messageHelper.methodParameters) + .replaceAll('@(message)', messageString) + ); + } + return messageHelper; + + case ST.placeholderExpr: + assert(node.children[1].type == ST.identifier); + final Node identifier = node.children[1]; + // Check that placeholders exist. + // TODO(thkim1011): Make message.placeholders a map so that we don't need to do linear time search. + // See https://github.com/flutter/flutter/issues/112709 + final Placeholder placeholder = message.placeholders.firstWhere( + (Placeholder placeholder) => placeholder.name == identifier.value, + orElse: () { + throw L10nException(''' +Make sure that the specified placeholder is defined in your arb file. +$translationForMessage +${Parser.indentForError(identifier.positionInMessage)}'''); + } + ); + dependentPlaceholders.add(placeholder); + return HelperMethod(dependentPlaceholders, placeholder: placeholder); + + case ST.pluralExpr: + requiresIntlImport = true; + final Map pluralLogicArgs = {}; + // Recall that pluralExpr are of the form + // pluralExpr := "{" ID "," "plural" "," pluralParts "}" + assert(node.children[1].type == ST.identifier); + assert(node.children[5].type == ST.pluralParts); + + final Node identifier = node.children[1]; + final Node pluralParts = node.children[5]; + + // Check that identifier exists and is of type int or num. + final Placeholder placeholder = message.placeholders.firstWhere( + (Placeholder placeholder) => placeholder.name == identifier.value, + orElse: () { + throw L10nException(''' +Make sure that the specified plural placeholder is defined in your arb file. +$translationForMessage +${List.filled(identifier.positionInMessage, ' ').join()}^'''); + } + ); + dependentPlaceholders.add(placeholder); + // TODO(thkim1011): Uncomment the following lines after Message refactor. + // See https://github.com/flutter/flutter/issues/112709. +// if (placeholder.type != 'num' && placeholder.type != 'int') { +// throw L10nException(''' +// The specified placeholder must be of type int or num. +// $translationForMessage +// ${List.filled(identifier.positionInMessage, ' ').join()}^'''); +// } + + for (final Node pluralPart in pluralParts.children.reversed) { + String pluralCase; + Node pluralMessage; + if (pluralPart.children[0].value == '=') { + assert(pluralPart.children[1].type == ST.number); + assert(pluralPart.children[3].type == ST.message); + pluralCase = pluralPart.children[1].value!; + pluralMessage = pluralPart.children[3]; + } else { + assert(pluralPart.children[0].type == ST.identifier || pluralPart.children[0].type == ST.other); + assert(pluralPart.children[2].type == ST.message); + pluralCase = pluralPart.children[0].value!; + pluralMessage = pluralPart.children[2]; + } + if (!pluralLogicArgs.containsKey(pluralCases[pluralCase])) { + final HelperMethod pluralPartHelper = generateHelperMethods(pluralMessage); + pluralLogicArgs[pluralCases[pluralCase]!] = ' ${pluralCases[pluralCase]}: ${pluralPartHelper.helperOrPlaceholder},'; + dependentPlaceholders.addAll(pluralPartHelper.dependentPlaceholders); + } else { + logger.printWarning(''' +The plural part specified below is overrided by a later plural part. +$translationForMessage +${Parser.indentForError(pluralPart.positionInMessage)} +'''); + } + } + final String helperMethodName = getHelperMethodName(); + final HelperMethod pluralHelper = HelperMethod(dependentPlaceholders, helper: helperMethodName); + helperMethods.add(pluralHelperTemplate + .replaceAll('@(name)', helperMethodName) + .replaceAll('@(parameters)', pluralHelper.methodParameters) + .replaceAll('@(count)', identifier.value!) + .replaceAll('@(pluralLogicArgs)', pluralLogicArgs.values.join('\n')) + ); + return pluralHelper; + + case ST.selectExpr: + requiresIntlImport = true; + // Recall that pluralExpr are of the form + // pluralExpr := "{" ID "," "plural" "," pluralParts "}" + assert(node.children[1].type == ST.identifier); + assert(node.children[5].type == ST.selectParts); + + final Node identifier = node.children[1]; + // Check that identifier exists + final Placeholder placeholder = message.placeholders.firstWhere( + (Placeholder placeholder) => placeholder.name == identifier.value, + orElse: () { + throw L10nException(''' +Make sure that the specified select placeholder is defined in your arb file. +$translationForMessage +${Parser.indentForError(identifier.positionInMessage)}'''); + } + ); + dependentPlaceholders.add(placeholder); + final List selectLogicArgs = []; + final Node selectParts = node.children[5]; + + for (final Node selectPart in selectParts.children) { + assert(selectPart.children[0].type == ST.identifier || selectPart.children[0].type == ST.other); + assert(selectPart.children[2].type == ST.message); + final String selectCase = selectPart.children[0].value!; + final Node selectMessage = selectPart.children[2]; + final HelperMethod selectPartHelper = generateHelperMethods(selectMessage); + selectLogicArgs.add(" '$selectCase': ${selectPartHelper.helperOrPlaceholder},"); + dependentPlaceholders.addAll(selectPartHelper.dependentPlaceholders); + } + final String helperMethodName = getHelperMethodName(); + final HelperMethod selectHelper = HelperMethod(dependentPlaceholders, helper: helperMethodName); + + helperMethods.add(selectHelperTemplate + .replaceAll('@(name)', helperMethodName) + .replaceAll('@(parameters)', selectHelper.methodParameters) + .replaceAll('@(choice)', identifier.value!) + .replaceAll('@(selectCases)', selectLogicArgs.join('\n')) + ); + return HelperMethod(dependentPlaceholders, helper: helperMethodName); + // ignore: no_default_cases + default: + throw Exception('Cannot call "generateHelperMethod" on node type ${node.type}'); + } + } + generateHelperMethods(node, isRoot: true); + return helperMethods.last.replaceAll('@(helperMethods)', helperMethods.sublist(0, helperMethods.length - 1).join('\n\n')); + } List writeOutputFiles({ bool isFromYaml = false }) { // First, generate the string contents of all necessary files. diff --git a/packages/flutter_tools/lib/src/localizations/gen_l10n_templates.dart b/packages/flutter_tools/lib/src/localizations/gen_l10n_templates.dart index f43b7c5b4872..c2bb67d84e1f 100644 --- a/packages/flutter_tools/lib/src/localizations/gen_l10n_templates.dart +++ b/packages/flutter_tools/lib/src/localizations/gen_l10n_templates.dart @@ -135,70 +135,37 @@ const String getterTemplate = ''' String get @(name) => @(message);'''; const String methodTemplate = ''' - @override - String @(name)(@(parameters)) { - return @(message); - }'''; - -const String formatMethodTemplate = ''' @override String @(name)(@(parameters)) { @(dateFormatting) @(numberFormatting) +@(helperMethods) return @(message); }'''; -const String pluralMethodTemplate = ''' - @override - String @(name)(@(parameters)) { -@(dateFormatting) -@(numberFormatting) - return intl.Intl.pluralLogic( - @(count), - locale: localeName, -@(pluralLogicArgs), - ); - }'''; - -const String pluralMethodTemplateInString = ''' - @override - String @(name)(@(parameters)) { -@(dateFormatting) -@(numberFormatting) - final String @(variable) = intl.Intl.pluralLogic( - @(count), - locale: localeName, -@(pluralLogicArgs), - ); - - return @(string); - }'''; - -const String selectMethodTemplate = ''' - @override - String @(name)(@(parameters)) { - return intl.Intl.select( - @(choice), - { - @(cases) - }, - desc: '@(description)' - ); - }'''; - -const String selectMethodTemplateInString = ''' - @override - String @(name)(@(parameters)) { - final String @(variable) = intl.Intl.select( - @(choice), - { - @(cases) - }, - desc: '@(description)' - ); - - return @(string); - }'''; +const String messageHelperTemplate = ''' + String @(name)(@(parameters)) { + return @(message); + }'''; + +const String pluralHelperTemplate = ''' + String @(name)(@(parameters)) { + return intl.Intl.pluralLogic( + @(count), + locale: localeName, +@(pluralLogicArgs) + ); + }'''; + +const String selectHelperTemplate = ''' + String @(name)(@(parameters)) { + return intl.Intl.selectLogic( + @(choice), + { +@(selectCases) + }, + ); + }'''; const String classFileTemplate = ''' @(header)@(requiresIntlImport)import '@(fileName)'; diff --git a/packages/flutter_tools/lib/src/localizations/gen_l10n_types.dart b/packages/flutter_tools/lib/src/localizations/gen_l10n_types.dart index 1209006c17ca..4f4ab6804134 100644 --- a/packages/flutter_tools/lib/src/localizations/gen_l10n_types.dart +++ b/packages/flutter_tools/lib/src/localizations/gen_l10n_types.dart @@ -129,6 +129,25 @@ class L10nException implements Exception { String toString() => message; } +class L10nParserException extends L10nException { + L10nParserException( + this.error, + this.fileName, + this.messageId, + this.messageString, + this.charNumber + ): super(''' +$error +[$fileName:$messageId] $messageString +${List.filled(4 + fileName.length + messageId.length + charNumber, ' ').join()}^'''); + + final String error; + final String fileName; + final String messageId; + final String messageString; + final int charNumber; +} + // One optional named parameter to be used by a NumberFormat. // // Some of the NumberFormat factory constructors have optional named parameters. @@ -202,16 +221,16 @@ class Placeholder { final String resourceId; final String name; final String? example; - final String? type; + String? type; final String? format; final List optionalParameters; final bool? isCustomDateFormat; - bool get requiresFormatting => ['DateTime', 'double', 'num'].contains(type) || (type == 'int' && format != null); - bool get isNumber => ['double', 'int', 'num'].contains(type); + bool get requiresFormatting => requiresDateFormatting || requiresNumFormatting; + bool get requiresDateFormatting => type == 'DateTime'; + bool get requiresNumFormatting => ['int', 'num', 'double'].contains(type) && format != null; bool get hasValidNumberFormat => _validNumberFormats.contains(format); bool get hasNumberFormatWithParameters => _numberFormatsWithNamedParameters.contains(format); - bool get isDate => 'DateTime' == type; bool get hasValidDateFormat => _validDateFormats.contains(format); static String? _stringAttribute( @@ -290,6 +309,8 @@ class Placeholder { // The value of this Message is "Hello World". The Message's value is the // localized string to be shown for the template ARB file's locale. // The docs for the Placeholder explain how placeholder entries are defined. +// TODO(thkim1011): We need to refactor this Message class to own all the messages in each language. +// See https://github.com/flutter/flutter/issues/112709. class Message { Message(Map bundle, this.resourceId, bool isResourceAttributeRequired) : assert(bundle != null), @@ -298,7 +319,12 @@ class Message { description = _description(bundle, resourceId, isResourceAttributeRequired), placeholders = _placeholders(bundle, resourceId, isResourceAttributeRequired), _pluralMatch = _pluralRE.firstMatch(_value(bundle, resourceId)), - _selectMatch = _selectRE.firstMatch(_value(bundle, resourceId)); + _selectMatch = _selectRE.firstMatch(_value(bundle, resourceId)) { + if (isPlural) { + final Placeholder placeholder = getCountPlaceholder(); + placeholder.type = 'num'; + } + } static final RegExp _pluralRE = RegExp(r'\s*\{([\w\s,]*),\s*plural\s*,'); static final RegExp _selectRE = RegExp(r'\s*\{([\w\s,]*),\s*select\s*,'); @@ -769,3 +795,50 @@ final Set _iso639Languages = { 'zh', 'zu', }; + +// Used in LocalizationsGenerator._generateMethod.generateHelperMethod. +class HelperMethod { + HelperMethod(this.dependentPlaceholders, {this.helper, this.placeholder, this.string }): + assert((() { + // At least one of helper, placeholder, string must be nonnull. + final bool a = helper == null; + final bool b = placeholder == null; + final bool c = string == null; + return (!a && b && c) || (a && !b && c) || (a && b && !c); + })()); + + Set dependentPlaceholders; + String? helper; + Placeholder? placeholder; + String? string; + + String get helperOrPlaceholder { + if (helper != null) { + return '$helper($methodArguments)'; + } else if (string != null) { + return '$string'; + } else { + if (placeholder!.requiresFormatting) { + return '${placeholder!.name}String'; + } else { + return placeholder!.name; + } + } + } + + String get methodParameters { + assert(helper != null); + return dependentPlaceholders.map((Placeholder placeholder) => + (placeholder.requiresFormatting) + ? 'String ${placeholder.name}String' + : '${placeholder.type} ${placeholder.name}').join(', '); + } + + String get methodArguments { + assert(helper != null); + return dependentPlaceholders.map((Placeholder placeholder) => + (placeholder.requiresFormatting) + ? '${placeholder.name}String' + : placeholder.name).join(', '); + } +} diff --git a/packages/flutter_tools/lib/src/localizations/localizations_utils.dart b/packages/flutter_tools/lib/src/localizations/localizations_utils.dart index d001c56026e4..b9e02808e063 100644 --- a/packages/flutter_tools/lib/src/localizations/localizations_utils.dart +++ b/packages/flutter_tools/lib/src/localizations/localizations_utils.dart @@ -292,7 +292,34 @@ String generateString(String value) { // Reintroduce escaped backslashes into generated Dart string. .replaceAll(backslash, r'\\'); - return "'$value'"; + return value; +} + +/// Given a list of strings, placeholders, or helper function calls, concatenate +/// them into one expression to be returned. +String generateReturnExpr(List helpers) { + if (helpers.isEmpty) { + return "''"; + } else if ( + helpers.length == 1 + && helpers[0].string == null + && (helpers[0].placeholder?.type == 'String' || helpers[0].helper != null) + ) { + return helpers[0].helperOrPlaceholder; + } else { + final String string = helpers.reversed.fold('', (String string, HelperMethod helper) { + if (helper.string != null) { + return generateString(helper.string!) + string; + } + final RegExp alphanumeric = RegExp(r'^([0-9a-zA-Z]|_)+$'); + if (alphanumeric.hasMatch(helper.helperOrPlaceholder) && !(string.isNotEmpty && alphanumeric.hasMatch(string[0]))) { + return '\$${helper.helperOrPlaceholder}$string'; + } else { + return '\${${helper.helperOrPlaceholder}}$string'; + } + }); + return "'$string'"; + } } /// Typed configuration from the localizations config file. diff --git a/packages/flutter_tools/lib/src/localizations/message_parser.dart b/packages/flutter_tools/lib/src/localizations/message_parser.dart new file mode 100644 index 000000000000..1ca0d42eb922 --- /dev/null +++ b/packages/flutter_tools/lib/src/localizations/message_parser.dart @@ -0,0 +1,543 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// The whole design for the lexing and parsing step can be found in this design doc. +// See https://flutter.dev/go/icu-message-parser. + +// Symbol Types +import 'gen_l10n_types.dart'; + +enum ST { + // Terminal Types + openBrace, + closeBrace, + comma, + equalSign, + other, + plural, + select, + string, + number, + identifier, + empty, + // Nonterminal Types + message, + + placeholderExpr, + + pluralExpr, + pluralParts, + pluralPart, + + selectExpr, + selectParts, + selectPart, +} + +// The grammar of the syntax. +Map>> grammar = >>{ + ST.message: >[ + [ST.string, ST.message], + [ST.placeholderExpr, ST.message], + [ST.pluralExpr, ST.message], + [ST.selectExpr, ST.message], + [ST.empty], + ], + ST.placeholderExpr: >[ + [ST.openBrace, ST.identifier, ST.closeBrace], + ], + ST.pluralExpr: >[ + [ST.openBrace, ST.identifier, ST.comma, ST.plural, ST.comma, ST.pluralParts, ST.closeBrace], + ], + ST.pluralParts: >[ + [ST.pluralPart, ST.pluralParts], + [ST.empty], + ], + ST.pluralPart: >[ + [ST.identifier, ST.openBrace, ST.message, ST.closeBrace], + [ST.equalSign, ST.number, ST.openBrace, ST.message, ST.closeBrace], + [ST.other, ST.openBrace, ST.message, ST.closeBrace], + ], + ST.selectExpr: >[ + [ST.openBrace, ST.identifier, ST.comma, ST.select, ST.comma, ST.selectParts, ST.closeBrace], + [ST.other, ST.openBrace, ST.message, ST.closeBrace], + ], + ST.selectParts: >[ + [ST.selectPart, ST.selectParts], + [ST.empty], + ], + ST.selectPart: >[ + [ST.identifier, ST.openBrace, ST.message, ST.closeBrace], + [ST.other, ST.openBrace, ST.message, ST.closeBrace], + ], +}; + +class Node { + Node(this.type, this.positionInMessage, { this.expectedSymbolCount = 0, this.value, List? children }): children = children ?? []; + + // Token constructors. + Node.openBrace(this.positionInMessage): type = ST.openBrace, value = '{'; + Node.closeBrace(this.positionInMessage): type = ST.closeBrace, value = '}'; + Node.brace(this.positionInMessage, String this.value) { + if (value == '{') { + type = ST.openBrace; + } else if (value == '}') { + type = ST.closeBrace; + } else { + // We should never arrive here. + throw L10nException('Provided value $value is not a brace.'); + } + } + Node.equalSign(this.positionInMessage): type = ST.equalSign, value = '='; + Node.comma(this.positionInMessage): type = ST.comma, value = ','; + Node.string(this.positionInMessage, String this.value): type = ST.string; + Node.number(this.positionInMessage, String this.value): type = ST.number; + Node.identifier(this.positionInMessage, String this.value): type = ST.identifier; + Node.pluralKeyword(this.positionInMessage): type = ST.plural, value = 'plural'; + Node.selectKeyword(this.positionInMessage): type = ST.select, value = 'select'; + Node.otherKeyword(this.positionInMessage): type = ST.other, value = 'other'; + Node.empty(this.positionInMessage): type = ST.empty, value = ''; + + String? value; + late ST type; + List children = []; + int positionInMessage; + int expectedSymbolCount = 0; + + @override + String toString() { + return _toStringHelper(0); + } + + String _toStringHelper(int indentLevel) { + final String indent = List.filled(indentLevel, ' ').join(); + if (children.isEmpty) { + return ''' +${indent}Node($type, $positionInMessage${value == null ? '' : ", value: '$value'"})'''; + } + final String childrenString = children.map((Node child) => child._toStringHelper(indentLevel + 1)).join(',\n'); + return ''' +${indent}Node($type, $positionInMessage${value == null ? '' : ", value: '$value'"}, children: [ +$childrenString, +$indent])'''; + } + + // Only used for testing. We don't compare expectedSymbolCount because + // it is an auxiliary member used during the parse function but doesn't + // have meaning after calling compress. + @override + // ignore: avoid_equals_and_hash_code_on_mutable_classes, hash_and_equals + bool operator==(covariant Node other) { + if(value != other.value + || type != other.type + || positionInMessage != other.positionInMessage + || children.length != other.children.length + ) { + return false; + } + for (int i = 0; i < children.length; i++) { + if (children[i] != other.children[i]) { + return false; + } + } + return true; + } + + bool get isFull { + return children.length >= expectedSymbolCount; + } +} + +RegExp unescapedString = RegExp(r'[^{}]+'); +RegExp brace = RegExp(r'{|}'); + +RegExp whitespace = RegExp(r'\s+'); +RegExp pluralKeyword = RegExp(r'plural'); +RegExp selectKeyword = RegExp(r'select'); +RegExp otherKeyword = RegExp(r'other'); +RegExp numeric = RegExp(r'[0-9]+'); +RegExp alphanumeric = RegExp(r'[a-zA-Z0-9]+'); +RegExp comma = RegExp(r','); +RegExp equalSign = RegExp(r'='); + +// List of token matchers ordered by precedence +Map matchers = { + ST.empty: whitespace, + ST.plural: pluralKeyword, + ST.select: selectKeyword, + ST.other: otherKeyword, + ST.number: numeric, + ST.comma: comma, + ST.equalSign: equalSign, + ST.identifier: alphanumeric, +}; + +class Parser { + Parser(this.messageId, this.filename, this.messageString); + + final String messageId; + final String messageString; + final String filename; + + static String indentForError(int position) { + return '${List.filled(position, ' ').join()}^'; + } + + // Lexes the message into a list of typed tokens. General idea is that + // every instance of "{" and "}" toggles the isString boolean and every + // instance of "'" toggles the isEscaped boolean (and treats a double + // single quote "''" as a single quote "'"). When !isString and !isEscaped + // delimit tokens by whitespace and special characters. + List lexIntoTokens() { + final List tokens = []; + bool isString = true; + // Index specifying where to match from + int startIndex = 0; + + // At every iteration, we should be able to match a new token until we + // reach the end of the string. If for some reason we don't match a + // token in any iteration of the loop, throw an error. + while (startIndex < messageString.length) { + Match? match; + if (isString) { + // TODO(thkim1011): Uncomment this when we add escaping as an option. + // See https://github.com/flutter/flutter/issues/113455. + // match = escapedString.matchAsPrefix(message, startIndex); + // if (match != null) { + // final String string = match.group(0)!; + // tokens.add(Node.string(startIndex, string == "''" ? "'" : string.substring(1, string.length - 1))); + // startIndex = match.end; + // continue; + // } + match = unescapedString.matchAsPrefix(messageString, startIndex); + if (match != null) { + tokens.add(Node.string(startIndex, match.group(0)!)); + startIndex = match.end; + continue; + } + match = brace.matchAsPrefix(messageString, startIndex); + if (match != null) { + tokens.add(Node.brace(startIndex, match.group(0)!)); + isString = false; + startIndex = match.end; + continue; + } + // Theoretically, we only reach this point because of unmatched single quotes because + // 1. If it begins with single quotes, then we match the longest string contained in single quotes. + // 2. If it begins with braces, then we match those braces. + // 3. Else the first character is neither single quote or brace so it is matched by RegExp "unescapedString" + throw L10nParserException( + 'ICU Lexing Error: Unmatched single quotes.', + filename, + messageId, + messageString, + startIndex, + ); + } else { + RegExp matcher; + ST? matchedType; + + // Try to match tokens until we succeed + for (matchedType in matchers.keys) { + matcher = matchers[matchedType]!; + match = matcher.matchAsPrefix(messageString, startIndex); + if (match != null) { + break; + } + } + + if (match == null) { + match = brace.matchAsPrefix(messageString, startIndex); + if (match != null) { + tokens.add(Node.brace(startIndex, match.group(0)!)); + isString = true; + startIndex = match.end; + continue; + } + // This should only happen when there are special characters we are unable to match. + throw L10nParserException( + 'ICU Lexing Error: Unexpected character.', + filename, + messageId, + messageString, + startIndex + ); + } else if (matchedType == ST.empty) { + // Do not add whitespace as a token. + startIndex = match.end; + continue; + } else { + tokens.add(Node(matchedType!, startIndex, value: match.group(0))); + startIndex = match.end; + continue; + } + } + } + return tokens; + } + + Node parseIntoTree() { + final List tokens = lexIntoTokens(); + final List parsingStack = [ST.message]; + final Node syntaxTree = Node(ST.empty, 0, expectedSymbolCount: 1); + final List treeTraversalStack = [syntaxTree]; + + // Helper function for parsing and constructing tree. + void parseAndConstructNode(ST nonterminal, int ruleIndex) { + final Node parent = treeTraversalStack.last; + final List grammarRule = grammar[nonterminal]![ruleIndex]; + + // When we run out of tokens, just use -1 to represent the last index. + final int positionInMessage = tokens.isNotEmpty ? tokens.first.positionInMessage : -1; + final Node node = Node(nonterminal, positionInMessage, expectedSymbolCount: grammarRule.length); + parsingStack.addAll(grammarRule.reversed); + + // For tree construction, add nodes to the parent until the parent has all + // all the children it is expecting. + parent.children.add(node); + if (parent.isFull) { + treeTraversalStack.removeLast(); + } + treeTraversalStack.add(node); + } + + while (parsingStack.isNotEmpty) { + final ST symbol = parsingStack.removeLast(); + + // Figure out which production rule to use. + switch(symbol) { + case ST.message: + if (tokens.isEmpty) { + parseAndConstructNode(ST.message, 4); + } else if (tokens[0].type == ST.closeBrace) { + parseAndConstructNode(ST.message, 4); + } else if (tokens[0].type == ST.string) { + parseAndConstructNode(ST.message, 0); + } else if (tokens[0].type == ST.openBrace) { + if (3 < tokens.length && tokens[3].type == ST.plural) { + parseAndConstructNode(ST.message, 2); + } else if (3 < tokens.length && tokens[3].type == ST.select) { + parseAndConstructNode(ST.message, 3); + } else { + parseAndConstructNode(ST.message, 1); + } + } else { + // Theoretically, we can never get here. + throw L10nException('ICU Syntax Error.'); + } + break; + case ST.placeholderExpr: + parseAndConstructNode(ST.placeholderExpr, 0); + break; + case ST.pluralExpr: + parseAndConstructNode(ST.pluralExpr, 0); + break; + case ST.pluralParts: + if (tokens.isNotEmpty && ( + tokens[0].type == ST.identifier || + tokens[0].type == ST.other || + tokens[0].type == ST.equalSign + ) + ) { + parseAndConstructNode(ST.pluralParts, 0); + } else { + parseAndConstructNode(ST.pluralParts, 1); + } + break; + case ST.pluralPart: + if (tokens.isNotEmpty && tokens[0].type == ST.identifier) { + parseAndConstructNode(ST.pluralPart, 0); + } else if (tokens.isNotEmpty && tokens[0].type == ST.equalSign) { + parseAndConstructNode(ST.pluralPart, 1); + } else if (tokens.isNotEmpty && tokens[0].type == ST.other) { + parseAndConstructNode(ST.pluralPart, 2); + } else { + throw L10nParserException( + 'ICU Syntax Error: Plural parts must be of the form "identifier { message }" or "= number { message }"', + filename, + messageId, + messageString, + tokens[0].positionInMessage, + ); + } + break; + case ST.selectExpr: + parseAndConstructNode(ST.selectExpr, 0); + break; + case ST.selectParts: + if (tokens.isNotEmpty && ( + tokens[0].type == ST.identifier || + tokens[0].type == ST.other + )) { + parseAndConstructNode(ST.selectParts, 0); + } else { + parseAndConstructNode(ST.selectParts, 1); + } + break; + case ST.selectPart: + if (tokens.isNotEmpty && tokens[0].type == ST.identifier) { + parseAndConstructNode(ST.selectPart, 0); + } else if (tokens.isNotEmpty && tokens[0].type == ST.other) { + parseAndConstructNode(ST.selectPart, 1); + } else { + throw L10nParserException( + 'ICU Syntax Error: Select parts must be of the form "identifier { message }"', + filename, + messageId, + messageString, + tokens[0].positionInMessage + ); + } + break; + // At this point, we are only handling terminal symbols. + // ignore: no_default_cases + default: + final Node parent = treeTraversalStack.last; + // If we match a terminal symbol, then remove it from tokens and + // add it to the tree. + if (symbol == ST.empty) { + parent.children.add(Node.empty(-1)); + } else if (tokens.isEmpty) { + throw L10nParserException( + 'ICU Syntax Error: Expected "${terminalTypeToString[symbol]}" but found no tokens.', + filename, + messageId, + messageString, + messageString.length + 1, + ); + } else if (symbol == tokens[0].type) { + final Node token = tokens.removeAt(0); + parent.children.add(token); + } else { + throw L10nParserException( + 'ICU Syntax Error: Expected "${terminalTypeToString[symbol]}" but found "${tokens[0].value}".', + filename, + messageId, + messageString, + tokens[0].positionInMessage, + ); + } + + if (parent.isFull) { + treeTraversalStack.removeLast(); + } + } + } + + return syntaxTree.children[0]; + } + + final Map terminalTypeToString = { + ST.openBrace: '{', + ST.closeBrace: '}', + ST.comma: ',', + ST.empty: '', + ST.identifier: 'identifier', + ST.number: 'number', + ST.plural: 'plural', + ST.select: 'select', + ST.equalSign: '=', + ST.other: 'other', + }; + + // Compress the syntax tree. Note that after + // parse(lex(message)), the individual parts (ST.string, ST.placeholderExpr, + // ST.pluralExpr, and ST.selectExpr) are structured as a linked list See diagram + // below. This + // function compresses these parts into a single children array (and does this + // for ST.pluralParts and ST.selectParts as well). Then it checks extra syntax + // rules. Essentially, it converts + // + // Message + // / \ + // PluralExpr Message + // / \ + // String Message + // / \ + // SelectExpr ... + // + // to + // + // Message + // / | \ + // PluralExpr String SelectExpr ... + // + // Keep in mind that this modifies the tree in place and the values of + // expectedSymbolCount and isFull is no longer useful after this operation. + Node compress(Node syntaxTree) { + Node node = syntaxTree; + final List children = []; + switch (syntaxTree.type) { + case ST.message: + case ST.pluralParts: + case ST.selectParts: + while (node.children.length == 2) { + children.add(node.children[0]); + compress(node.children[0]); + node = node.children[1]; + } + syntaxTree.children = children; + break; + // ignore: no_default_cases + default: + node.children.forEach(compress); + } + return syntaxTree; + } + + // Takes in a compressed syntax tree and checks extra rules on + // plural parts and select parts. + void checkExtraRules(Node syntaxTree) { + final List children = syntaxTree.children; + switch(syntaxTree.type) { + case ST.pluralParts: + // Must have an "other" case. + if (children.every((Node node) => node.children[0].type != ST.other)) { + throw L10nParserException( + 'ICU Syntax Error: Plural expressions must have an "other" case.', + filename, + messageId, + messageString, + syntaxTree.positionInMessage + ); + } + // Identifier must be one of "zero", "one", "two", "few", "many". + for (final Node node in children) { + final Node pluralPartFirstToken = node.children[0]; + const List validIdentifiers = ['zero', 'one', 'two', 'few', 'many']; + if (pluralPartFirstToken.type == ST.identifier && !validIdentifiers.contains(pluralPartFirstToken.value)) { + throw L10nParserException( + 'ICU Syntax Error: Plural expressions case must be one of "zero", "one", "two", "few", "many", or "other".', + filename, + messageId, + messageString, + node.positionInMessage, + ); + } + } + break; + case ST.selectParts: + if (children.every((Node node) => node.children[0].type != ST.other)) { + throw L10nParserException( + 'ICU Syntax Error: Select expressions must have an "other" case.', + filename, + messageId, + messageString, + syntaxTree.positionInMessage, + ); + } + break; + // ignore: no_default_cases + default: + break; + } + children.forEach(checkExtraRules); + } + + Node parse() { + final Node syntaxTree = compress(parseIntoTree()); + checkExtraRules(syntaxTree); + return syntaxTree; + } +} diff --git a/packages/flutter_tools/test/general.shard/generate_localizations_test.dart b/packages/flutter_tools/test/general.shard/generate_localizations_test.dart index 38d57607fe84..37010fe62ef4 100644 --- a/packages/flutter_tools/test/general.shard/generate_localizations_test.dart +++ b/packages/flutter_tools/test/general.shard/generate_localizations_test.dart @@ -1580,6 +1580,51 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e expect(localizationsFile, contains('output-localization-file_en.loadLibrary()')); }); + group('placeholder tests', () { + testWithoutContext('should throw attempting to generate a select message without placeholders', () { + const String selectMessageWithoutPlaceholdersAttribute = ''' +{ + "helloWorld": "Hello {name}", + "@helloWorld": { + "description": "Improperly formatted since it has no placeholder attribute.", + "placeholders": { + "hello": {}, + "world": {} + } + } +}'''; + + final Directory l10nDirectory = fs.currentDirectory.childDirectory('lib').childDirectory('l10n') + ..createSync(recursive: true); + l10nDirectory.childFile(defaultTemplateArbFileName) + .writeAsStringSync(selectMessageWithoutPlaceholdersAttribute); + + expect( + () { + LocalizationsGenerator( + fileSystem: fs, + inputPathString: defaultL10nPathString, + outputPathString: defaultL10nPathString, + templateArbFileName: defaultTemplateArbFileName, + outputFileString: defaultOutputFileString, + classNameString: defaultClassNameString, + logger: logger, + ) + ..loadResources() + ..writeOutputFiles(); + }, + throwsA(isA().having( + (L10nException e) => e.message, + 'message', + contains(''' +Make sure that the specified placeholder is defined in your arb file. +Hello {name} + ^'''), + )), + ); + }); + }); + group('DateTime tests', () { testWithoutContext('imports package:intl', () { const String singleDateMessageArbFileString = ''' @@ -1895,7 +1940,13 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e throwsA(isA().having( (L10nException e) => e.message, 'message', - contains('Check to see if the plural message is in the proper ICU syntax format'), + // TODO(thkim1011): Uncomment after work on refactoring the Message class. + // See https://github.com/flutter/flutter/issues/112709. +// contains(''' +// Make sure that the specified plural placeholder is defined in your arb file. +// {count,plural, =0{Hello}=1{Hello World}=2{Hello two worlds}few{Hello {count} worlds}many{Hello all {count} worlds}other{Hello other {count} worlds}} +// ^'''), + contains('Cannot find the count placeholder in plural message "helloWorlds".'), )), ); }); @@ -1932,7 +1983,13 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e throwsA(isA().having( (L10nException e) => e.message, 'message', - contains('Check to see if the plural message is in the proper ICU syntax format'), + // TODO(thkim1011): Uncomment after work on refactoring the Message class. + // See https://github.com/flutter/flutter/issues/112709. +// contains(''' +// Make sure that the specified plural placeholder is defined in your arb file. +// {count,plural, =0{Hello}=1{Hello World}=2{Hello two worlds}few{Hello {count} worlds}many{Hello all {count} worlds}other{Hello other {count} worlds}} +// ^'''), + contains('Cannot find the count placeholder in plural message "helloWorlds".'), )), ); }); @@ -1965,7 +2022,13 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e throwsA(isA().having( (L10nException e) => e.message, 'message', - contains('Resource attribute "@helloWorlds" was not found'), + // TODO(thkim1011): Uncomment after work on refactoring the Message class. + // See https://github.com/flutter/flutter/issues/112709. +// contains(''' +// Make sure that the specified plural placeholder is defined in your arb file. +// {count,plural, =0{Hello}=1{Hello World}=2{Hello two worlds}few{Hello {count} worlds}many{Hello all {count} worlds}other{Hello other {count} worlds}} +// ^'''), + contains('Resource attribute "@helloWorlds" was not found. Please ensure that plural resources have a corresponding @resource.'), )), ); }); @@ -2008,36 +2071,6 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e )), ); }); - - testWithoutContext('should warn attempting to generate a plural message whose placeholder is not num or null', () { - const String pluralMessageWithIncorrectPlaceholderType = ''' -{ - "helloWorlds": "{count,plural, =0{Hello}=1{Hello World}=2{Hello two worlds}few{Hello {count} worlds}many{Hello all {count} worlds}other{Hello other {count} worlds}}", - "@helloWorlds": { - "placeholders": { - "count": { - "type": "int" - } - } - } -}'''; - final Directory l10nDirectory = fs.currentDirectory.childDirectory('lib').childDirectory('l10n') - ..createSync(recursive: true); - l10nDirectory.childFile(defaultTemplateArbFileName) - .writeAsStringSync(pluralMessageWithIncorrectPlaceholderType); - LocalizationsGenerator( - fileSystem: fs, - inputPathString: defaultL10nPathString, - outputPathString: defaultL10nPathString, - templateArbFileName: defaultTemplateArbFileName, - outputFileString: defaultOutputFileString, - classNameString: defaultClassNameString, - logger: logger, - ) - ..loadResources() - ..writeOutputFiles(); - expect(logger.warningText, contains("Placeholders for plurals are automatically converted to type 'num'")); - }); }); group('select messages', () { @@ -2072,7 +2105,10 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e throwsA(isA().having( (L10nException e) => e.message, 'message', - contains('Check to see if the select message is in the proper ICU syntax format'), + contains(''' +Make sure that the specified select placeholder is defined in your arb file. +{gender, select, female {She} male {He} other {they} } + ^'''), )), ); }); @@ -2109,7 +2145,10 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e throwsA(isA().having( (L10nException e) => e.message, 'message', - contains('Check to see if the select message is in the proper ICU syntax format'), + contains(''' +Make sure that the specified select placeholder is defined in your arb file. +{gender, select, female {She} male {He} other {they} } + ^'''), )), ); }); @@ -2142,7 +2181,13 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e throwsA(isA().having( (L10nException e) => e.message, 'message', - contains('Resource attribute "@genderSelect" was not found'), + // TODO(thkim1011): Uncomment after work on refactoring the Message class. + // See https://github.com/flutter/flutter/issues/112709. +// contains(''' +// Make sure that the specified select placeholder is defined in your arb file. +// {gender, select, female {She} male {He} other {they} } +// ^'''), + contains('Resource attribute "@genderSelect" was not found. Please ensure that select resources have a corresponding @resource.'), )), ); }); @@ -2219,10 +2264,10 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e throwsA(isA().having( (L10nException e) => e.message, 'message', - allOf( - contains('Incorrect select message format for'), - contains('Check to see if the select message is in the proper ICU syntax format.'), - ), + contains(''' +Select expressions must have an "other" case. +[app_en.arb:genderSelect] {gender, select,} + ^'''), )), ); }); @@ -2543,27 +2588,27 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e expect(localizationsFile, contains(r'${six}m')); expect(localizationsFile, contains(r'$seven')); expect(localizationsFile, contains(r'$eight')); - expect(localizationsFile, contains(r'${nine}')); + expect(localizationsFile, contains(r'$nine')); }); testWithoutContext('check for string interpolation rules - plurals', () { const String enArbCheckList = ''' { - "first": "{count,plural, =0{test {count} test} =1{哈{count}哈} =2{m{count}m} few{_{count}_} many{{count} test} other{{count}m}", + "first": "{count,plural, =0{test {count} test} =1{哈{count}哈} =2{m{count}m} few{_{count}_} many{{count} test} other{{count}m}}", "@first": { "description": "First set of plural messages to test.", "placeholders": { "count": {} } }, - "second": "{count,plural, =0{test {count}} other{ {count}}", + "second": "{count,plural, =0{test {count}} other{ {count}}}", "@second": { "description": "Second set of plural messages to test.", "placeholders": { "count": {} } }, - "third": "{total,plural, =0{test {total}} other{ {total}}", + "third": "{total,plural, =0{test {total}} other{ {total}}}", "@third": { "description": "Third set of plural messages to test, for number.", "placeholders": { @@ -2580,8 +2625,8 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e // generated code for use of '${variable}' vs '$variable' const String esArbCheckList = ''' { - "first": "{count,plural, =0{test {count} test} =1{哈{count}哈} =2{m{count}m} few{_{count}_} many{{count} test} other{{count}m}", - "second": "{count,plural, =0{test {count}} other{ {count}}" + "first": "{count,plural, =0{test {count} test} =1{哈{count}哈} =2{m{count}m} few{_{count}_} many{{count} test} other{{count}m}}", + "second": "{count,plural, =0{test {count}} other{ {count}}}" } '''; @@ -2614,8 +2659,8 @@ import 'output-localization-file_en.dart' deferred as output-localization-file_e expect(localizationsFile, contains(r'test $count')); expect(localizationsFile, contains(r' $count')); expect(localizationsFile, contains(r'String totalString = totalNumberFormat')); - expect(localizationsFile, contains(r'test $totalString')); - expect(localizationsFile, contains(r' $totalString')); + expect(localizationsFile, contains(r'totalString')); + expect(localizationsFile, contains(r'totalString')); }); testWithoutContext( @@ -2994,4 +3039,38 @@ AppLocalizations lookupAppLocalizations(Locale locale) { expect(localizationsFile, containsIgnoringWhitespace(r'String tryToPollute(num count) {')); expect(localizationsFile, containsIgnoringWhitespace(r'String withoutType(num count) {')); }); + + // TODO(thkim1011): Uncomment when implementing escaping. + // See https://github.com/flutter/flutter/issues/113455. +// testWithoutContext('escaping with single quotes', () { +// const String arbFile = ''' +// { +// "singleQuote": "Flutter''s amazing!", +// "@singleQuote": { +// "description": "A message with a single quote." +// } +// }'''; + +// final Directory l10nDirectory = fs.currentDirectory.childDirectory('lib').childDirectory('l10n') +// ..createSync(recursive: true); +// l10nDirectory.childFile(defaultTemplateArbFileName) +// .writeAsStringSync(arbFile); + +// LocalizationsGenerator( +// fileSystem: fs, +// inputPathString: defaultL10nPathString, +// outputPathString: defaultL10nPathString, +// templateArbFileName: defaultTemplateArbFileName, +// outputFileString: defaultOutputFileString, +// classNameString: defaultClassNameString, +// logger: logger, +// ) +// ..loadResources() +// ..writeOutputFiles(); + +// final String localizationsFile = fs.file( +// fs.path.join(syntheticL10nPackagePath, 'output-localization-file_en.dart'), +// ).readAsStringSync(); +// expect(localizationsFile, contains(r"Flutter\'s amazing")); +// }); } diff --git a/packages/flutter_tools/test/general.shard/message_parser_test.dart b/packages/flutter_tools/test/general.shard/message_parser_test.dart new file mode 100644 index 000000000000..48825bc4521d --- /dev/null +++ b/packages/flutter_tools/test/general.shard/message_parser_test.dart @@ -0,0 +1,491 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_tools/src/localizations/gen_l10n_types.dart'; +import 'package:flutter_tools/src/localizations/message_parser.dart'; +import '../src/common.dart'; + +void main() { + // Going to test that operator== is overloaded properly since the rest + // of the test depends on it. + testWithoutContext('node equality', () { + final Node actual = Node( + ST.placeholderExpr, + 0, + expectedSymbolCount: 3, + children: [ + Node.openBrace(0), + Node.string(1, 'var'), + Node.closeBrace(4), + ], + ); + + final Node expected = Node( + ST.placeholderExpr, + 0, + expectedSymbolCount: 3, + children: [ + Node.openBrace(0), + Node.string(1, 'var'), + Node.closeBrace(4), + ], + ); + expect(actual, equals(expected)); + + final Node wrongType = Node( + ST.pluralExpr, + 0, + expectedSymbolCount: 3, + children: [ + Node.openBrace(0), + Node.string(1, 'var'), + Node.closeBrace(4), + ], + ); + expect(actual, isNot(equals(wrongType))); + + final Node wrongPosition = Node( + ST.placeholderExpr, + 1, + expectedSymbolCount: 3, + children: [ + Node.openBrace(0), + Node.string(1, 'var'), + Node.closeBrace(4), + ], + ); + expect(actual, isNot(equals(wrongPosition))); + + final Node wrongChildrenCount = Node( + ST.placeholderExpr, + 0, + expectedSymbolCount: 3, + children: [ + Node.string(1, 'var'), + Node.closeBrace(4), + ], + ); + expect(actual, isNot(equals(wrongChildrenCount))); + + final Node wrongChild = Node( + ST.placeholderExpr, + 0, + expectedSymbolCount: 3, + children: [ + Node.closeBrace(0), + Node.string(1, 'var'), + Node.closeBrace(4), + ], + ); + expect(actual, isNot(equals(wrongChild))); + }); + + testWithoutContext('lexer basic', () { + final List tokens1 = Parser( + 'helloWorld', + 'app_en.arb', + 'Hello {name}' + ).lexIntoTokens(); + expect(tokens1, equals([ + Node.string(0, 'Hello '), + Node.openBrace(6), + Node.identifier(7, 'name'), + Node.closeBrace(11), + ])); + + final List tokens2 = Parser( + 'plural', + 'app_en.arb', + 'There are {count} {count, plural, =1{cat} other{cats}}' + ).lexIntoTokens(); + expect(tokens2, equals([ + Node.string(0, 'There are '), + Node.openBrace(10), + Node.identifier(11, 'count'), + Node.closeBrace(16), + Node.string(17, ' '), + Node.openBrace(18), + Node.identifier(19, 'count'), + Node.comma(24), + Node.pluralKeyword(26), + Node.comma(32), + Node.equalSign(34), + Node.number(35, '1'), + Node.openBrace(36), + Node.string(37, 'cat'), + Node.closeBrace(40), + Node.otherKeyword(42), + Node.openBrace(47), + Node.string(48, 'cats'), + Node.closeBrace(52), + Node.closeBrace(53), + ])); + + final List tokens3 = Parser( + 'gender', + 'app_en.arb', + '{gender, select, male{he} female{she} other{they}}' + ).lexIntoTokens(); + expect(tokens3, equals([ + Node.openBrace(0), + Node.identifier(1, 'gender'), + Node.comma(7), + Node.selectKeyword(9), + Node.comma(15), + Node.identifier(17, 'male'), + Node.openBrace(21), + Node.string(22, 'he'), + Node.closeBrace(24), + Node.identifier(26, 'female'), + Node.openBrace(32), + Node.string(33, 'she'), + Node.closeBrace(36), + Node.otherKeyword(38), + Node.openBrace(43), + Node.string(44, 'they'), + Node.closeBrace(48), + Node.closeBrace(49), + ])); + }); + + testWithoutContext('lexer recursive', () { + final List tokens = Parser( + 'plural', + 'app_en.arb', + '{count, plural, =1{{gender, select, male{he} female{she}}} other{they}}' + ).lexIntoTokens(); + expect(tokens, equals([ + Node.openBrace(0), + Node.identifier(1, 'count'), + Node.comma(6), + Node.pluralKeyword(8), + Node.comma(14), + Node.equalSign(16), + Node.number(17, '1'), + Node.openBrace(18), + Node.openBrace(19), + Node.identifier(20, 'gender'), + Node.comma(26), + Node.selectKeyword(28), + Node.comma(34), + Node.identifier(36, 'male'), + Node.openBrace(40), + Node.string(41, 'he'), + Node.closeBrace(43), + Node.identifier(45, 'female'), + Node.openBrace(51), + Node.string(52, 'she'), + Node.closeBrace(55), + Node.closeBrace(56), + Node.closeBrace(57), + Node.otherKeyword(59), + Node.openBrace(64), + Node.string(65, 'they'), + Node.closeBrace(69), + Node.closeBrace(70), + ])); + }); + + // TODO(thkim1011): Uncomment when implementing escaping. + // See https://github.com/flutter/flutter/issues/113455. + // testWithoutContext('lexer escaping', () { + // final List tokens1 = Parser("''").lexIntoTokens(); + // expect(tokens1, equals([Node.string(0, "'")])); + + // final List tokens2 = Parser("'hello world { name }'").lexIntoTokens(); + // expect(tokens2, equals([Node.string(0, 'hello world { name }')])); + + // final List tokens3 = Parser("'{ escaped string }' { not escaped }").lexIntoTokens(); + // expect(tokens3, equals([ + // Node.string(0, '{ escaped string }'), + // Node.string(20, ' '), + // Node.openBrace(21), + // Node.identifier(23, 'not'), + // Node.identifier(27, 'escaped'), + // Node.closeBrace(35), + // ])); + + // final List tokens4 = Parser("Flutter''s amazing!").lexIntoTokens(); + // expect(tokens4, equals([ + // Node.string(0, 'Flutter'), + // Node.string(7, "'"), + // Node.string(9, 's amazing!'), + // ])); + // }); + + testWithoutContext('lexer: lexically correct but syntactically incorrect', () { + final List tokens = Parser( + 'syntax', + 'app_en.arb', + 'string { identifier { string { identifier } } }' + ).lexIntoTokens(); + expect(tokens, equals([ + Node.string(0, 'string '), + Node.openBrace(7), + Node.identifier(9, 'identifier'), + Node.openBrace(20), + Node.string(21, ' string '), + Node.openBrace(29), + Node.identifier(31, 'identifier'), + Node.closeBrace(42), + Node.string(43, ' '), + Node.closeBrace(44), + Node.closeBrace(46), + ])); + }); + + // TODO(thkim1011): Uncomment when implementing escaping. + // See https://github.com/flutter/flutter/issues/113455. +// testWithoutContext('lexer unmatched single quote', () { +// const String message = "here''s an unmatched single quote: '"; +// const String expectedError = ''' +// ICU Lexing Error: Unmatched single quotes. +// here''s an unmatched single quote: ' +// ^'''; +// expect( +// () => Parser(message).lexIntoTokens(), +// throwsA(isA().having( +// (L10nException e) => e.message, +// 'message', +// contains(expectedError), +// ))); +// }); + + testWithoutContext('lexer unexpected character', () { + const String message = '{ * }'; + const String expectedError = ''' +ICU Lexing Error: Unexpected character. +[app_en.arb:lex] { * } + ^'''; + expect( + () => Parser('lex', 'app_en.arb', message).lexIntoTokens(), + throwsA(isA().having( + (L10nException e) => e.message, + 'message', + contains(expectedError), + ))); + }); + + + testWithoutContext('parser basic', () { + expect(Parser('helloWorld', 'app_en.arb', 'Hello {name}').parse(), equals( + Node(ST.message, 0, children: [ + Node(ST.string, 0, value: 'Hello '), + Node(ST.placeholderExpr, 6, children: [ + Node(ST.openBrace, 6, value: '{'), + Node(ST.identifier, 7, value: 'name'), + Node(ST.closeBrace, 11, value: '}') + ]) + ]) + )); + + expect(Parser( + 'plural', + 'app_en.arb', + 'There are {count} {count, plural, =1{cat} other{cats}}' + ).parse(), equals( + Node(ST.message, 0, children: [ + Node(ST.string, 0, value: 'There are '), + Node(ST.placeholderExpr, 10, children: [ + Node(ST.openBrace, 10, value: '{'), + Node(ST.identifier, 11, value: 'count'), + Node(ST.closeBrace, 16, value: '}') + ]), + Node(ST.string, 17, value: ' '), + Node(ST.pluralExpr, 18, children: [ + Node(ST.openBrace, 18, value: '{'), + Node(ST.identifier, 19, value: 'count'), + Node(ST.comma, 24, value: ','), + Node(ST.plural, 26, value: 'plural'), + Node(ST.comma, 32, value: ','), + Node(ST.pluralParts, 34, children: [ + Node(ST.pluralPart, 34, children: [ + Node(ST.equalSign, 34, value: '='), + Node(ST.number, 35, value: '1'), + Node(ST.openBrace, 36, value: '{'), + Node(ST.message, 37, children: [ + Node(ST.string, 37, value: 'cat') + ]), + Node(ST.closeBrace, 40, value: '}') + ]), + Node(ST.pluralPart, 42, children: [ + Node(ST.other, 42, value: 'other'), + Node(ST.openBrace, 47, value: '{'), + Node(ST.message, 48, children: [ + Node(ST.string, 48, value: 'cats') + ]), + Node(ST.closeBrace, 52, value: '}') + ]) + ]), + Node(ST.closeBrace, 53, value: '}') + ]), + ]), + )); + + expect(Parser( + 'gender', + 'app_en.arb', + '{gender, select, male{he} female{she} other{they}}' + ).parse(), equals( + Node(ST.message, 0, children: [ + Node(ST.selectExpr, 0, children: [ + Node(ST.openBrace, 0, value: '{'), + Node(ST.identifier, 1, value: 'gender'), + Node(ST.comma, 7, value: ','), + Node(ST.select, 9, value: 'select'), + Node(ST.comma, 15, value: ','), + Node(ST.selectParts, 17, children: [ + Node(ST.selectPart, 17, children: [ + Node(ST.identifier, 17, value: 'male'), + Node(ST.openBrace, 21, value: '{'), + Node(ST.message, 22, children: [ + Node(ST.string, 22, value: 'he'), + ]), + Node(ST.closeBrace, 24, value: '}'), + ]), + Node(ST.selectPart, 26, children: [ + Node(ST.identifier, 26, value: 'female'), + Node(ST.openBrace, 32, value: '{'), + Node(ST.message, 33, children: [ + Node(ST.string, 33, value: 'she'), + ]), + Node(ST.closeBrace, 36, value: '}'), + ]), + Node(ST.selectPart, 38, children: [ + Node(ST.other, 38, value: 'other'), + Node(ST.openBrace, 43, value: '{'), + Node(ST.message, 44, children: [ + Node(ST.string, 44, value: 'they'), + ]), + Node(ST.closeBrace, 48, value: '}'), + ]), + ]), + Node(ST.closeBrace, 49, value: '}'), + ]), + ]) + )); + }); + + // TODO(thkim1011): Uncomment when implementing escaping. + // See https://github.com/flutter/flutter/issues/113455. + // testWithoutContext('parser basic 2', () { + // expect(Parser("Flutter''s amazing!").parse(), equals( + // Node(ST.message, 0, children: [ + // Node(ST.string, 0, value: 'Flutter'), + // Node(ST.string, 7, value: "'"), + // Node(ST.string, 9, value: 's amazing!'), + // ]) + // )); + // }); + + testWithoutContext('parser recursive', () { + expect(Parser( + 'pluralGender', + 'app_en.arb', + '{count, plural, =1{{gender, select, male{he} female{she} other{they}}} other{they}}' + ).parse(), equals( + Node(ST.message, 0, children: [ + Node(ST.pluralExpr, 0, children: [ + Node(ST.openBrace, 0, value: '{'), + Node(ST.identifier, 1, value: 'count'), + Node(ST.comma, 6, value: ','), + Node(ST.plural, 8, value: 'plural'), + Node(ST.comma, 14, value: ','), + Node(ST.pluralParts, 16, children: [ + Node(ST.pluralPart, 16, children: [ + Node(ST.equalSign, 16, value: '='), + Node(ST.number, 17, value: '1'), + Node(ST.openBrace, 18, value: '{'), + Node(ST.message, 19, children: [ + Node(ST.selectExpr, 19, children: [ + Node(ST.openBrace, 19, value: '{'), + Node(ST.identifier, 20, value: 'gender'), + Node(ST.comma, 26, value: ','), + Node(ST.select, 28, value: 'select'), + Node(ST.comma, 34, value: ','), + Node(ST.selectParts, 36, children: [ + Node(ST.selectPart, 36, children: [ + Node(ST.identifier, 36, value: 'male'), + Node(ST.openBrace, 40, value: '{'), + Node(ST.message, 41, children: [ + Node(ST.string, 41, value: 'he'), + ]), + Node(ST.closeBrace, 43, value: '}'), + ]), + Node(ST.selectPart, 45, children: [ + Node(ST.identifier, 45, value: 'female'), + Node(ST.openBrace, 51, value: '{'), + Node(ST.message, 52, children: [ + Node(ST.string, 52, value: 'she'), + ]), + Node(ST.closeBrace, 55, value: '}'), + ]), + Node(ST.selectPart, 57, children: [ + Node(ST.other, 57, value: 'other'), + Node(ST.openBrace, 62, value: '{'), + Node(ST.message, 63, children: [ + Node(ST.string, 63, value: 'they'), + ]), + Node(ST.closeBrace, 67, value: '}'), + ]), + ]), + Node(ST.closeBrace, 68, value: '}'), + ]), + ]), + Node(ST.closeBrace, 69, value: '}'), + ]), + Node(ST.pluralPart, 71, children: [ + Node(ST.other, 71, value: 'other'), + Node(ST.openBrace, 76, value: '{'), + Node(ST.message, 77, children: [ + Node(ST.string, 77, value: 'they'), + ]), + Node(ST.closeBrace, 81, value: '}'), + ]), + ]), + Node(ST.closeBrace, 82, value: '}'), + ]), + ]) + )); + }); + + testWithoutContext('parser unexpected token', () { + // unexpected token + const String expectedError1 = ''' +ICU Syntax Error: Expected "}" but found "=". +[app_en.arb:unexpectedToken] { placeholder = + ^'''; + expect( + () => Parser('unexpectedToken', 'app_en.arb', '{ placeholder =').parse(), + throwsA(isA().having( + (L10nException e) => e.message, + 'message', + contains(expectedError1), + ))); + + const String expectedError2 = ''' +ICU Syntax Error: Expected "number" but found "}". +[app_en.arb:unexpectedToken] { count, plural, = } + ^'''; + expect( + () => Parser('unexpectedToken', 'app_en.arb', '{ count, plural, = }').parse(), + throwsA(isA().having( + (L10nException e) => e.message, + 'message', + contains(expectedError2), + ))); + + const String expectedError3 = ''' +ICU Syntax Error: Expected "identifier" but found ",". +[app_en.arb:unexpectedToken] { , plural , = } + ^'''; + expect( + () => Parser('unexpectedToken', 'app_en.arb', '{ , plural , = }').parse(), + throwsA(isA().having( + (L10nException e) => e.message, + 'message', + contains(expectedError3), + ))); + }); +} diff --git a/packages/flutter_tools/test/integration.shard/gen_l10n_test.dart b/packages/flutter_tools/test/integration.shard/gen_l10n_test.dart index 077e7ab24cd2..6827b3814a17 100644 --- a/packages/flutter_tools/test/integration.shard/gen_l10n_test.dart +++ b/packages/flutter_tools/test/integration.shard/gen_l10n_test.dart @@ -123,45 +123,48 @@ void main() { '#l10n 70 (Indeed, they like Flutter!)\n' '#l10n 71 (Indeed, he likes ice cream!)\n' '#l10n 72 (Indeed, she likes chocolate!)\n' - '#l10n 73 (--- es ---)\n' - '#l10n 74 (ES - Hello world)\n' - '#l10n 75 (ES - Hello _NEWLINE_ World)\n' - '#l10n 76 (ES - Hola \$ Mundo)\n' - '#l10n 77 (ES - Hello Mundo)\n' - '#l10n 78 (ES - Hola Mundo)\n' - '#l10n 79 (ES - Hello World on viernes, 1 de enero de 1960)\n' - '#l10n 80 (ES - Hello world argument on 1/1/1960 at 0:00)\n' - '#l10n 81 (ES - Hello World from 1960 to 2020)\n' - '#l10n 82 (ES - Hello for 123)\n' - '#l10n 83 (ES - Hello)\n' - '#l10n 84 (ES - Hello World)\n' - '#l10n 85 (ES - Hello two worlds)\n' + '#l10n 73 (he)\n' + '#l10n 74 (they)\n' + '#l10n 75 (she)\n' + '#l10n 76 (--- es ---)\n' + '#l10n 77 (ES - Hello world)\n' + '#l10n 78 (ES - Hello _NEWLINE_ World)\n' + '#l10n 79 (ES - Hola \$ Mundo)\n' + '#l10n 80 (ES - Hello Mundo)\n' + '#l10n 81 (ES - Hola Mundo)\n' + '#l10n 82 (ES - Hello World on viernes, 1 de enero de 1960)\n' + '#l10n 83 (ES - Hello world argument on 1/1/1960 at 0:00)\n' + '#l10n 84 (ES - Hello World from 1960 to 2020)\n' + '#l10n 85 (ES - Hello for 123)\n' '#l10n 86 (ES - Hello)\n' - '#l10n 87 (ES - Hello nuevo World)\n' - '#l10n 88 (ES - Hello two nuevo worlds)\n' - '#l10n 89 (ES - Hello on viernes, 1 de enero de 1960)\n' - '#l10n 90 (ES - Hello World, on viernes, 1 de enero de 1960)\n' - '#l10n 91 (ES - Hello two worlds, on viernes, 1 de enero de 1960)\n' - '#l10n 92 (ES - Hello other 0 worlds, with a total of 100 citizens)\n' - '#l10n 93 (ES - Hello World of 101 citizens)\n' - '#l10n 94 (ES - Hello two worlds with 102 total citizens)\n' - '#l10n 95 (ES - [Hola] -Mundo- #123#)\n' - '#l10n 96 (ES - \$!)\n' - '#l10n 97 (ES - One \$)\n' - "#l10n 98 (ES - Flutter's amazing!)\n" - "#l10n 99 (ES - Flutter's amazing, times 2!)\n" - '#l10n 100 (ES - Flutter is "amazing"!)\n' - '#l10n 101 (ES - Flutter is "amazing", times 2!)\n' - '#l10n 102 (ES - 16 wheel truck)\n' - "#l10n 103 (ES - Sedan's elegance)\n" - '#l10n 104 (ES - Cabriolet has "acceleration")\n' - '#l10n 105 (ES - Oh, she found ES - 1 itemES - !)\n' - '#l10n 106 (ES - Indeed, ES - they like ES - Flutter!)\n' - '#l10n 107 (--- es_419 ---)\n' - '#l10n 108 (ES 419 - Hello World)\n' - '#l10n 109 (ES 419 - Hello)\n' - '#l10n 110 (ES 419 - Hello World)\n' - '#l10n 111 (ES 419 - Hello two worlds)\n' + '#l10n 87 (ES - Hello World)\n' + '#l10n 88 (ES - Hello two worlds)\n' + '#l10n 89 (ES - Hello)\n' + '#l10n 90 (ES - Hello nuevo World)\n' + '#l10n 91 (ES - Hello two nuevo worlds)\n' + '#l10n 92 (ES - Hello on viernes, 1 de enero de 1960)\n' + '#l10n 93 (ES - Hello World, on viernes, 1 de enero de 1960)\n' + '#l10n 94 (ES - Hello two worlds, on viernes, 1 de enero de 1960)\n' + '#l10n 95 (ES - Hello other 0 worlds, with a total of 100 citizens)\n' + '#l10n 96 (ES - Hello World of 101 citizens)\n' + '#l10n 97 (ES - Hello two worlds with 102 total citizens)\n' + '#l10n 98 (ES - [Hola] -Mundo- #123#)\n' + '#l10n 99 (ES - \$!)\n' + '#l10n 100 (ES - One \$)\n' + "#l10n 101 (ES - Flutter's amazing!)\n" + "#l10n 102 (ES - Flutter's amazing, times 2!)\n" + '#l10n 103 (ES - Flutter is "amazing"!)\n' + '#l10n 104 (ES - Flutter is "amazing", times 2!)\n' + '#l10n 105 (ES - 16 wheel truck)\n' + "#l10n 106 (ES - Sedan's elegance)\n" + '#l10n 107 (ES - Cabriolet has "acceleration")\n' + '#l10n 108 (ES - Oh, she found ES - 1 itemES - !)\n' + '#l10n 109 (ES - Indeed, ES - they like ES - Flutter!)\n' + '#l10n 110 (--- es_419 ---)\n' + '#l10n 111 (ES 419 - Hello World)\n' + '#l10n 112 (ES 419 - Hello)\n' + '#l10n 113 (ES 419 - Hello World)\n' + '#l10n 114 (ES 419 - Hello two worlds)\n' '#l10n END\n' ); } diff --git a/packages/flutter_tools/test/integration.shard/test_data/gen_l10n_project.dart b/packages/flutter_tools/test/integration.shard/test_data/gen_l10n_project.dart index 02e7045f0154..0723fe5b7235 100644 --- a/packages/flutter_tools/test/integration.shard/test_data/gen_l10n_project.dart +++ b/packages/flutter_tools/test/integration.shard/test_data/gen_l10n_project.dart @@ -229,6 +229,9 @@ class Home extends StatelessWidget { "${localizations.selectInString('he')}", "${localizations.selectWithPlaceholder('male', 'ice cream')}", "${localizations.selectWithPlaceholder('female', 'chocolate')}", + "${localizations.selectInPlural('male', 1)}", + "${localizations.selectInPlural('male', 2)}", + "${localizations.selectInPlural('female', 1)}", ]); }, ), @@ -627,7 +630,7 @@ void main() { } }, - "singleQuoteSelect": "{vehicleType, select, sedan{Sedan's elegance} cabriolet{Cabriolet' acceleration} truck{truck's heavy duty} other{Other's mirrors!}}", + "singleQuoteSelect": "{vehicleType, select, sedan{Sedan's elegance} cabriolet{Cabriolet's acceleration} truck{truck's heavy duty} other{Other's mirrors!}}", "@singleQuoteSelect": { "description": "A select message with a single quote.", "placeholders": { @@ -666,6 +669,19 @@ void main() { "gender": {}, "preference": {} } + }, + + "selectInPlural": "{count, plural, =1{{gender, select, male{he} female{she} other{they}}} other{they}}", + "@selectInPlural": { + "description": "Pronoun dependent on the count and gender.", + "placeholders": { + "gender": { + "type": "String" + }, + "count": { + "type": "num" + } + } } } '''; @@ -702,7 +718,7 @@ void main() { "helloFor": "ES - Hello for {value}", "helloAdjectiveWorlds": "{count,plural, =0{ES - Hello} =1{ES - Hello {adjective} World} =2{ES - Hello two {adjective} worlds} other{ES - Hello other {count} {adjective} worlds}}", "helloWorldsOn": "{count,plural, =0{ES - Hello on {date}} =1{ES - Hello World, on {date}} =2{ES - Hello two worlds, on {date}} other{ES - Hello other {count} worlds, on {date}}}", - "helloWorldPopulation": "{ES - count,plural, =1{ES - Hello World of {population} citizens} =2{ES - Hello two worlds with {population} total citizens} many{ES - Hello all {count} worlds, with a total of {population} citizens} other{ES - Hello other {count} worlds, with a total of {population} citizens}}", + "helloWorldPopulation": "{count,plural, =1{ES - Hello World of {population} citizens} =2{ES - Hello two worlds with {population} total citizens} many{ES - Hello all {count} worlds, with a total of {population} citizens} other{ES - Hello other {count} worlds, with a total of {population} citizens}}", "helloWorldInterpolation": "ES - [{hello}] #{world}#", "helloWorldsInterpolation": "{count,plural, other {ES - [{hello}] -{world}- #{count}#}}", "dollarSign": "ES - $!",