From 21885142146e377b29f1acefa0a92db4a4b370be Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Tue, 22 Feb 2022 15:10:02 +0100 Subject: [PATCH] [YAML] Update scripts/tools/zap_regen_all.py to be able to generate only the test targets if needed (#15246) * [YAML] Update chip-tool templates in such a way that tests can be generated alone * [YAML] Update src/darwin templates in such a way that tests can be generated alone * [YAML] Update examples/placeholder dedicated tests templates with the new paths * Update scripts/tools/zap_regen_all.py to be able to generate only the test targets if needed --- examples/chip-tool/templates/helper.js | 6 -- examples/chip-tool/templates/templates.json | 21 +---- .../commands.zapt} | 0 examples/chip-tool/templates/tests/helper.js | 26 ++++++ .../partials/command_value.zapt} | 0 .../{ => tests}/partials/test_cluster.zapt | 0 .../partials/value_equals.zapt} | 0 .../chip-tool/templates/tests/templates.json | 40 +++++++++ .../chip-tool/templates/{ => tests}/tests.js | 0 examples/placeholder/templates/templates.json | 14 +-- scripts/tools/zap_regen_all.py | 87 +++++++++++++++---- src/darwin/Framework/CHIP/templates/helper.js | 6 -- .../Framework/CHIP/templates/templates.json | 24 +---- .../CHIPClustersTests.zapt} | 0 .../Framework/CHIP/templates/tests/helper.js | 35 ++++++++ .../partials/check_test_value.zapt | 0 .../{ => tests}/partials/defined_value.zapt | 0 .../{ => tests}/partials/test_cluster.zapt | 0 .../{ => tests}/partials/test_value.zapt | 0 .../CHIP/templates/tests/templates.json | 45 ++++++++++ .../CHIP/templates/{ => tests}/tests.js | 0 21 files changed, 219 insertions(+), 85 deletions(-) rename examples/chip-tool/templates/{tests-commands.zapt => tests/commands.zapt} (100%) create mode 100644 examples/chip-tool/templates/tests/helper.js rename examples/chip-tool/templates/{partials/test_cluster_command_value.zapt => tests/partials/command_value.zapt} (100%) rename examples/chip-tool/templates/{ => tests}/partials/test_cluster.zapt (100%) rename examples/chip-tool/templates/{partials/test_cluster_value_equals.zapt => tests/partials/value_equals.zapt} (100%) create mode 100644 examples/chip-tool/templates/tests/templates.json rename examples/chip-tool/templates/{ => tests}/tests.js (100%) rename src/darwin/Framework/CHIP/templates/{clusters-tests.zapt => tests/CHIPClustersTests.zapt} (100%) create mode 100644 src/darwin/Framework/CHIP/templates/tests/helper.js rename src/darwin/Framework/CHIP/templates/{ => tests}/partials/check_test_value.zapt (100%) rename src/darwin/Framework/CHIP/templates/{ => tests}/partials/defined_value.zapt (100%) rename src/darwin/Framework/CHIP/templates/{ => tests}/partials/test_cluster.zapt (100%) rename src/darwin/Framework/CHIP/templates/{ => tests}/partials/test_value.zapt (100%) create mode 100644 src/darwin/Framework/CHIP/templates/tests/templates.json rename src/darwin/Framework/CHIP/templates/{ => tests}/tests.js (100%) diff --git a/examples/chip-tool/templates/helper.js b/examples/chip-tool/templates/helper.js index daf3bd5c5ba56c..aa778c8a645718 100644 --- a/examples/chip-tool/templates/helper.js +++ b/examples/chip-tool/templates/helper.js @@ -104,11 +104,6 @@ function asTypeMaxValue(type) return templateUtil.templatePromise(this.global, promise); } -function utf8StringLength(str) -{ - return new TextEncoder().encode(str).length -} - async function structs_with_cluster_name(options) { const packageId = await templateUtil.ensureZclPackageId(this); @@ -140,5 +135,4 @@ async function structs_with_cluster_name(options) exports.asDelimitedCommand = asDelimitedCommand; exports.asTypeMinValue = asTypeMinValue; exports.asTypeMaxValue = asTypeMaxValue; -exports.utf8StringLength = utf8StringLength; exports.structs_with_cluster_name = structs_with_cluster_name; diff --git a/examples/chip-tool/templates/templates.json b/examples/chip-tool/templates/templates.json index 8a39d5b2da4800..e61ce2120942d7 100644 --- a/examples/chip-tool/templates/templates.json +++ b/examples/chip-tool/templates/templates.json @@ -6,9 +6,7 @@ "../../../src/app/zap-templates/common/StringHelper.js", "../../../src/app/zap-templates/templates/app/helper.js", "../../../src/app/zap-templates/templates/chip/helper.js", - "../../../src/app/zap-templates/common/ClusterTestGeneration.js", - "helper.js", - "tests.js" + "helper.js" ], "override": "../../../src/app/zap-templates/common/override.js", "partials": [ @@ -23,18 +21,6 @@ { "name": "cluster_header", "path": "../../../src/app/zap-templates/partials/cluster_header.zapt" - }, - { - "name": "test_cluster", - "path": "partials/test_cluster.zapt" - }, - { - "name": "commandValue", - "path": "partials/test_cluster_command_value.zapt" - }, - { - "name": "valueEquals", - "path": "partials/test_cluster_value_equals.zapt" } ], "templates": [ @@ -62,11 +48,6 @@ "path": "ComplexArgumentParser-src.zapt", "name": "Complex Argument Parser", "output": "cluster/ComplexArgumentParser.cpp" - }, - { - "path": "tests-commands.zapt", - "name": "Tests Commands header", - "output": "test/Commands.h" } ] } diff --git a/examples/chip-tool/templates/tests-commands.zapt b/examples/chip-tool/templates/tests/commands.zapt similarity index 100% rename from examples/chip-tool/templates/tests-commands.zapt rename to examples/chip-tool/templates/tests/commands.zapt diff --git a/examples/chip-tool/templates/tests/helper.js b/examples/chip-tool/templates/tests/helper.js new file mode 100644 index 00000000000000..93e938357870b0 --- /dev/null +++ b/examples/chip-tool/templates/tests/helper.js @@ -0,0 +1,26 @@ +/* + * + * Copyright (c) 2022 Project CHIP Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function utf8StringLength(str) +{ + return new TextEncoder().encode(str).length +} + +// +// Module exports +// +exports.utf8StringLength = utf8StringLength; diff --git a/examples/chip-tool/templates/partials/test_cluster_command_value.zapt b/examples/chip-tool/templates/tests/partials/command_value.zapt similarity index 100% rename from examples/chip-tool/templates/partials/test_cluster_command_value.zapt rename to examples/chip-tool/templates/tests/partials/command_value.zapt diff --git a/examples/chip-tool/templates/partials/test_cluster.zapt b/examples/chip-tool/templates/tests/partials/test_cluster.zapt similarity index 100% rename from examples/chip-tool/templates/partials/test_cluster.zapt rename to examples/chip-tool/templates/tests/partials/test_cluster.zapt diff --git a/examples/chip-tool/templates/partials/test_cluster_value_equals.zapt b/examples/chip-tool/templates/tests/partials/value_equals.zapt similarity index 100% rename from examples/chip-tool/templates/partials/test_cluster_value_equals.zapt rename to examples/chip-tool/templates/tests/partials/value_equals.zapt diff --git a/examples/chip-tool/templates/tests/templates.json b/examples/chip-tool/templates/tests/templates.json new file mode 100644 index 00000000000000..d06cce5a219428 --- /dev/null +++ b/examples/chip-tool/templates/tests/templates.json @@ -0,0 +1,40 @@ +{ + "name": "CHIP Tool Tests templates", + "version": "chip-v1", + "helpers": [ + "../../../../src/app/zap-templates/partials/helper.js", + "../../../../src/app/zap-templates/common/StringHelper.js", + "../../../../src/app/zap-templates/templates/app/helper.js", + "../../../../src/app/zap-templates/templates/chip/helper.js", + "../../../../src/app/zap-templates/common/ClusterTestGeneration.js", + "../helper.js", + "helper.js", + "tests.js" + ], + "override": "../../../../src/app/zap-templates/common/override.js", + "partials": [ + { + "name": "header", + "path": "../../../../src/app/zap-templates/partials/header.zapt" + }, + { + "name": "test_cluster", + "path": "partials/test_cluster.zapt" + }, + { + "name": "commandValue", + "path": "partials/command_value.zapt" + }, + { + "name": "valueEquals", + "path": "partials/value_equals.zapt" + } + ], + "templates": [ + { + "path": "commands.zapt", + "name": "Tests Commands header", + "output": "test/Commands.h" + } + ] +} diff --git a/examples/chip-tool/templates/tests.js b/examples/chip-tool/templates/tests/tests.js similarity index 100% rename from examples/chip-tool/templates/tests.js rename to examples/chip-tool/templates/tests/tests.js diff --git a/examples/placeholder/templates/templates.json b/examples/placeholder/templates/templates.json index f08de0822f95fa..2633e857238e84 100644 --- a/examples/placeholder/templates/templates.json +++ b/examples/placeholder/templates/templates.json @@ -16,25 +16,17 @@ "name": "header", "path": "../../../src/app/zap-templates/partials/header.zapt" }, - { - "name": "clusters_header", - "path": "../../../src/app/zap-templates/partials/clusters_header.zapt" - }, - { - "name": "cluster_header", - "path": "../../../src/app/zap-templates/partials/cluster_header.zapt" - }, { "name": "test_cluster", - "path": "../../../examples/chip-tool/templates/partials/test_cluster.zapt" + "path": "../../../examples/chip-tool/templates/tests/partials/test_cluster.zapt" }, { "name": "commandValue", - "path": "../../../examples/chip-tool/templates/partials/test_cluster_command_value.zapt" + "path": "../../../examples/chip-tool/templates/tests/partials/command_value.zapt" }, { "name": "valueEquals", - "path": "../../../examples/chip-tool/templates/partials/test_cluster_value_equals.zapt" + "path": "../../../examples/chip-tool/templates/tests/partials/value_equals.zapt" } ], "templates": [ diff --git a/scripts/tools/zap_regen_all.py b/scripts/tools/zap_regen_all.py index eb19dab5089989..90ebd0a22aece8 100755 --- a/scripts/tools/zap_regen_all.py +++ b/scripts/tools/zap_regen_all.py @@ -15,6 +15,7 @@ # limitations under the License. # +import argparse import os from pathlib import Path import sys @@ -47,6 +48,8 @@ def generate(self): cmd.append(self.template) if self.output_dir: + if not os.path.exists(self.output_dir): + os.makedirs(self.output_dir) cmd.append('-o') cmd.append(self.output_dir) @@ -61,6 +64,16 @@ def checkPythonVersion(): exit(1) +def setupArgumentsParser(): + parser = argparse.ArgumentParser( + description='Generate content from ZAP files') + parser.add_argument('--type', default='all', choices=['all', 'tests'], + help='Choose which content type to generate (default: all)') + parser.add_argument('--tests', default='all', choices=['all', 'chip-tool', 'darwin', 'app1', 'app2'], + help='When generating tests only target, Choose which tests to generate (default: all)') + return parser.parse_args() + + def getGlobalTemplatesTargets(): targets = [] @@ -72,7 +85,8 @@ def getGlobalTemplatesTargets(): # Place holder has apps within each build if example_name == "placeholder": example_name = filepath.as_posix() - example_name = example_name[example_name.index('apps/') + 5:] + example_name = example_name[example_name.index( + 'apps/') + len('apps/'):] example_name = example_name[:example_name.index('/')] logging.info("Found example %s (via %s)" % (example_name, str(filepath))) @@ -81,8 +95,6 @@ def getGlobalTemplatesTargets(): # a name like output_dir = os.path.join( 'zzz_generated', 'placeholder', example_name, 'zap-generated') - if not os.path.exists(output_dir): - os.makedirs(output_dir) template = 'examples/placeholder/templates/templates.json' targets.append(ZAPGenerateTarget(filepath, output_dir=output_dir)) @@ -97,9 +109,6 @@ def getGlobalTemplatesTargets(): # a name like output_dir = os.path.join( 'zzz_generated', example_name, 'zap-generated') - if not os.path.exists(output_dir): - os.makedirs(output_dir) - targets.append(ZAPGenerateTarget(filepath, output_dir=output_dir)) targets.append(ZAPGenerateTarget( @@ -109,8 +118,46 @@ def getGlobalTemplatesTargets(): return targets -def getSpecificTemplatesTargets(): +def getTestsTemplatesTargets(test_target): + templates = { + 'chip-tool': { + 'zap': 'src/controller/data_model/controller-clusters.zap', + 'template': 'examples/chip-tool/templates/tests/templates.json', + 'output_dir': 'zzz_generated/chip-tool/zap-generated' + }, + 'darwin': { + 'zap': 'src/controller/data_model/controller-clusters.zap', + 'template': 'src/darwin/Framework/CHIP/templates/tests/templates.json', + 'output_dir': None + } + } + + # Place holder has apps within each build + for filepath in Path('./examples/placeholder').rglob('*.zap'): + example_name = filepath.as_posix() + example_name = example_name[example_name.index( + 'apps/') + len('apps/'):] + example_name = example_name[:example_name.index('/')] + + templates[example_name] = { + 'zap': filepath, + 'template': 'examples/placeholder/templates/templates.json', + 'output_dir': os.path.join('zzz_generated', 'placeholder', example_name, 'zap-generated') + } + targets = [] + for key, target in templates.items(): + if test_target == 'all' or test_target == key: + logging.info("Found test target %s (via %s)" % + (key, target['template'])) + targets.append(ZAPGenerateTarget( + target['zap'], template=target['template'], output_dir=target['output_dir'])) + + return targets + + +def getSpecificTemplatesTargets(): + zap_filepath = 'src/controller/data_model/controller-clusters.zap' # Mapping of required template and output directory templates = { @@ -122,23 +169,24 @@ def getSpecificTemplatesTargets(): 'src/app/tests/suites/templates/templates.json': 'zzz_generated/controller-clusters/zap-generated', } + targets = [] for template, output_dir in templates.items(): - target = ZAPGenerateTarget( - 'src/controller/data_model/controller-clusters.zap', template=template) - if output_dir is not None: - if not os.path.exists(output_dir): - os.makedirs(output_dir) - target.output_dir = output_dir - - targets.append(target) + targets.append(ZAPGenerateTarget( + zap_filepath, template=template, output_dir=output_dir)) return targets -def getTargets(): +def getTargets(type, test_target): targets = [] - targets.extend(getGlobalTemplatesTargets()) - targets.extend(getSpecificTemplatesTargets()) + + if type == 'all': + targets.extend(getGlobalTemplatesTargets()) + targets.extend(getTestsTemplatesTargets('all')) + targets.extend(getSpecificTemplatesTargets()) + elif type == 'tests': + targets.extend(getTestsTemplatesTargets(test_target)) + return targets @@ -149,8 +197,9 @@ def main(): ) checkPythonVersion() os.chdir(CHIP_ROOT_DIR) + args = setupArgumentsParser() - targets = getTargets() + targets = getTargets(args.type, args.tests) for target in targets: target.generate() diff --git a/src/darwin/Framework/CHIP/templates/helper.js b/src/darwin/Framework/CHIP/templates/helper.js index d018a659a92f3b..7c0202555b72f6 100644 --- a/src/darwin/Framework/CHIP/templates/helper.js +++ b/src/darwin/Framework/CHIP/templates/helper.js @@ -79,11 +79,6 @@ function asObjectiveCNumberType(label, type, asLowerCased) return templateUtil.templatePromise(this.global, promise) } -function asTestIndex(index) -{ - return index.toString().padStart(6, 0); -} - async function asObjectiveCClass(type, cluster, options) { let pkgId = await templateUtil.ensureZclPackageId(this); @@ -155,7 +150,6 @@ function commandHasRequiredField(command) // exports.asObjectiveCBasicType = asObjectiveCBasicType; exports.asObjectiveCNumberType = asObjectiveCNumberType; -exports.asTestIndex = asTestIndex; exports.asObjectiveCClass = asObjectiveCClass; exports.asObjectiveCType = asObjectiveCType; exports.asStructPropertyName = asStructPropertyName; diff --git a/src/darwin/Framework/CHIP/templates/templates.json b/src/darwin/Framework/CHIP/templates/templates.json index 663edcb78046e5..f90bc62a5e1cd3 100644 --- a/src/darwin/Framework/CHIP/templates/templates.json +++ b/src/darwin/Framework/CHIP/templates/templates.json @@ -8,8 +8,7 @@ "../../../../../src/app/zap-templates/templates/app/helper.js", "../../../../../src/app/zap-templates/templates/chip/helper.js", "../../../../../src/app/zap-templates/common/ClusterTestGeneration.js", - "helper.js", - "tests.js" + "helper.js" ], "override": "../../../../../src/app/zap-templates/common/override.js", "partials": [ @@ -17,10 +16,6 @@ "name": "header", "path": "../../../../../src/app/zap-templates/partials/header.zapt" }, - { - "name": "test_cluster", - "path": "partials/test_cluster.zapt" - }, { "name": "CHIPCallbackBridge", "path": "partials/CHIPCallbackBridge.zapt" @@ -33,14 +28,6 @@ "name": "decode_value", "path": "partials/decode_value.zapt" }, - { - "name": "test_value", - "path": "partials/test_value.zapt" - }, - { - "name": "check_test_value", - "path": "partials/check_test_value.zapt" - }, { "name": "init_struct_member", "path": "partials/init_struct_member.zapt" @@ -52,10 +39,6 @@ { "name": "attribute_data_callback_name", "path": "partials/attribute_data_callback_name.zapt" - }, - { - "name": "defined_value", - "path": "partials/defined_value.zapt" } ], "templates": [ @@ -118,11 +101,6 @@ "path": "CHIPAttributeTLVValueDecoder-src.zapt", "name": "Decode TLV attribute values into Objc objects", "output": "src/darwin/Framework/CHIP/zap-generated/CHIPAttributeTLVValueDecoder.mm" - }, - { - "path": "clusters-tests.zapt", - "name": "Cluster Tests", - "output": "src/darwin/Framework/CHIPTests/CHIPClustersTests.m" } ] } diff --git a/src/darwin/Framework/CHIP/templates/clusters-tests.zapt b/src/darwin/Framework/CHIP/templates/tests/CHIPClustersTests.zapt similarity index 100% rename from src/darwin/Framework/CHIP/templates/clusters-tests.zapt rename to src/darwin/Framework/CHIP/templates/tests/CHIPClustersTests.zapt diff --git a/src/darwin/Framework/CHIP/templates/tests/helper.js b/src/darwin/Framework/CHIP/templates/tests/helper.js new file mode 100644 index 00000000000000..b188e85b9656b4 --- /dev/null +++ b/src/darwin/Framework/CHIP/templates/tests/helper.js @@ -0,0 +1,35 @@ +/* + * + * Copyright (c) 2022 Project CHIP Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * In order to run the tests into an expected order, an index is + * added to the generated test step. This is because tests using + * the XCTest framework are expected to be independent and should + * normally run as independent steps. But since the current code is + * dealing with a simulated accessory running into its own process + * for the whole duration of the tests, its state is not resetted + * between each steps which defeats XCTest expectation. + */ +function asTestIndex(index) +{ + return index.toString().padStart(6, 0); +} + +// +// Module exports +// +exports.asTestIndex = asTestIndex; diff --git a/src/darwin/Framework/CHIP/templates/partials/check_test_value.zapt b/src/darwin/Framework/CHIP/templates/tests/partials/check_test_value.zapt similarity index 100% rename from src/darwin/Framework/CHIP/templates/partials/check_test_value.zapt rename to src/darwin/Framework/CHIP/templates/tests/partials/check_test_value.zapt diff --git a/src/darwin/Framework/CHIP/templates/partials/defined_value.zapt b/src/darwin/Framework/CHIP/templates/tests/partials/defined_value.zapt similarity index 100% rename from src/darwin/Framework/CHIP/templates/partials/defined_value.zapt rename to src/darwin/Framework/CHIP/templates/tests/partials/defined_value.zapt diff --git a/src/darwin/Framework/CHIP/templates/partials/test_cluster.zapt b/src/darwin/Framework/CHIP/templates/tests/partials/test_cluster.zapt similarity index 100% rename from src/darwin/Framework/CHIP/templates/partials/test_cluster.zapt rename to src/darwin/Framework/CHIP/templates/tests/partials/test_cluster.zapt diff --git a/src/darwin/Framework/CHIP/templates/partials/test_value.zapt b/src/darwin/Framework/CHIP/templates/tests/partials/test_value.zapt similarity index 100% rename from src/darwin/Framework/CHIP/templates/partials/test_value.zapt rename to src/darwin/Framework/CHIP/templates/tests/partials/test_value.zapt diff --git a/src/darwin/Framework/CHIP/templates/tests/templates.json b/src/darwin/Framework/CHIP/templates/tests/templates.json new file mode 100644 index 00000000000000..b095c453edab58 --- /dev/null +++ b/src/darwin/Framework/CHIP/templates/tests/templates.json @@ -0,0 +1,45 @@ +{ + "name": "Framework Tests templates", + "version": "chip-v1", + "helpers": [ + "../../../../../../src/app/zap-templates/partials/helper.js", + "../../../../../../src/app/zap-templates/common/ChipTypesHelper.js", + "../../../../../../src/app/zap-templates/common/StringHelper.js", + "../../../../../../src/app/zap-templates/templates/app/helper.js", + "../../../../../../src/app/zap-templates/templates/chip/helper.js", + "../../../../../../src/app/zap-templates/common/ClusterTestGeneration.js", + "../helper.js", + "helper.js", + "tests.js" + ], + "override": "../../../../../../src/app/zap-templates/common/override.js", + "partials": [ + { + "name": "header", + "path": "../../../../../../src/app/zap-templates/partials/header.zapt" + }, + { + "name": "test_cluster", + "path": "partials/test_cluster.zapt" + }, + { + "name": "test_value", + "path": "partials/test_value.zapt" + }, + { + "name": "check_test_value", + "path": "partials/check_test_value.zapt" + }, + { + "name": "defined_value", + "path": "partials/defined_value.zapt" + } + ], + "templates": [ + { + "path": "CHIPClustersTests.zapt", + "name": "Cluster Tests", + "output": "src/darwin/Framework/CHIPTests/CHIPClustersTests.m" + } + ] +} diff --git a/src/darwin/Framework/CHIP/templates/tests.js b/src/darwin/Framework/CHIP/templates/tests/tests.js similarity index 100% rename from src/darwin/Framework/CHIP/templates/tests.js rename to src/darwin/Framework/CHIP/templates/tests/tests.js