diff --git a/.github/workflows/zap_regeneration.yaml b/.github/workflows/zap_regeneration.yaml index db0b2730a651e3..11f9492abdd16d 100644 --- a/.github/workflows/zap_regeneration.yaml +++ b/.github/workflows/zap_regeneration.yaml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: connectedhomeip/chip-build-zap:0.6.06 + image: connectedhomeip/chip-build:0.6.11 defaults: run: shell: sh @@ -44,17 +44,11 @@ jobs: token: ${{ github.token }} attempt_limit: 3 attempt_delay: 2000 - - name: Setup ZAP - timeout-minutes: 5 - run: | - cd third_party/zap/repo/ - npm ci - npm run version-stamp - npm rebuild canvas --update-binary - npm run build-spa - name: Generate all timeout-minutes: 5 run: scripts/tools/zap_regen_all.py + - name: Ensure git works in current working directory + run: git config --global --add safe.directory `pwd` - name: Add uncommitted changes run: git add . - name: Fix upstream diff --git a/.github/workflows/zap_templates.yaml b/.github/workflows/zap_templates.yaml index 6bb30919015855..ba21cfc103a1b1 100644 --- a/.github/workflows/zap_templates.yaml +++ b/.github/workflows/zap_templates.yaml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: connectedhomeip/chip-build-zap:0.6.06 + image: connectedhomeip/chip-build:0.6.11 defaults: run: shell: sh @@ -44,18 +44,10 @@ jobs: token: ${{ github.token }} attempt_limit: 3 attempt_delay: 2000 - - name: Checkout submodules - run: scripts/checkout_submodules.py --shallow --platform linux - - name: Setup ZAP - timeout-minutes: 10 - run: | - cd third_party/zap/repo/ - npm ci - npm run version-stamp - npm rebuild canvas --update-binary - npm run build-spa - name: Generate all run: scripts/tools/zap_regen_all.py + - name: Ensure git works in current working directory + run: git config --global --add safe.directory `pwd` - name: Check for uncommited changes run: | git add . diff --git a/.gitmodules b/.gitmodules index 0da2aaa3ce495a..a68fbf7fc095da 100644 --- a/.gitmodules +++ b/.gitmodules @@ -54,10 +54,6 @@ url = https://github.com/Qorvo/QMatter branch = vlatest-libs platforms = qpg -[submodule "zap"] - path = third_party/zap/repo - url = https://github.com/project-chip/zap.git - branch = master [submodule "freertos"] path = third_party/freertos/repo url = https://github.com/FreeRTOS/FreeRTOS-Kernel.git diff --git a/docs/guides/BUILDING.md b/docs/guides/BUILDING.md index 10fe92958e4b55..fa5c8817416979 100644 --- a/docs/guides/BUILDING.md +++ b/docs/guides/BUILDING.md @@ -128,6 +128,35 @@ update_config=1 Finally, reboot your RPi. +## Installing ZAP + +`zap-cli` is already installed in pre-built docker images for chip-build, such +as +[chip-build-vscode](https://hub.docker.com/r/connectedhomeip/chip-build-vscode). + +Zap generation and tooling relies on `zap-cli` being available on the current +system. You can install it from the zap project +[Releases](https://github.com/project-chip/zap/releases). + +You should install a compatible release version, generally checking against the +release set in +[integrations/docker/images/chip-build/Dockerfile](../../../integrations/docker/images/chip-build/Dockerfile). + +On linux, installation from `zap-linux.zip` is recommended as it pulls fewer +dependencies than the `.deb` package. + +### Which ZAP to use + +ZAP scripting uses the following detection, in order: + +- `$ZAP_DEVELOPMENT_PATH` to point to a zap checkout. Use this if you are + developing zap locally and would like to run zap with your changes + +- `$ZAP_INSTALL_PATH` to point to where `zap-linux.zip`/`zap-mac.zip` was + unpacked. This allows you to not need to place zap/zap-cli in `$PATH` + +- Otherwise scripts assume `zap-cli` or `zap` is available in `$PATH` + ## Prepare for building Before running any other build command, the `scripts/activate.sh` environment diff --git a/examples/chef/chef.py b/examples/chef/chef.py index e3a81564cfa230..f6b3cc385897a9 100755 --- a/examples/chef/chef.py +++ b/examples/chef/chef.py @@ -299,8 +299,6 @@ def main() -> int: action="store_true", dest="do_interact") parser.add_option("-m", "--menuconfig", help="runs menuconfig on platforms that support it", action="store_true", dest="do_menuconfig") - parser.add_option("", "--bootstrap_zap", help="installs zap dependencies", - action="store_true", dest="do_bootstrap_zap") parser.add_option("-z", "--zap", help="runs zap to generate data model & interaction model artifacts", action="store_true", dest="do_run_zap") parser.add_option("-g", "--zapgui", help="runs zap GUI display to allow editing of data model", @@ -354,29 +352,6 @@ def main() -> int: splash() - # - # ZAP bootstrapping - # - - if options.do_bootstrap_zap: - if sys.platform == "linux" or sys.platform == "linux2": - flush_print("Installing ZAP OS package dependencies") - install_deps_cmd = """\ - sudo apt-get install nodejs node-yargs npm - libpixman-1-dev libcairo2-dev libpango1.0-dev node-pre-gyp - libjpeg9-dev libgif-dev node-typescript""" - shell.run_cmd(unwrap_cmd(install_deps_cmd)) - if sys.platform == "darwin": - flush_print( - "Installation of ZAP OS packages not supported on MacOS") - if sys.platform == "win32": - flush_print( - "Installation of ZAP OS packages not supported on Windows") - - flush_print("Running NPM to install ZAP Node.JS dependencies") - shell.run_cmd( - f"cd {_REPO_BASE_PATH}/third_party/zap/repo/ && npm install") - # # CI # diff --git a/integrations/docker/images/chip-cert-bins/Dockerfile b/integrations/docker/images/chip-cert-bins/Dockerfile index 2aa6f10e767fb0..14fce86b771cd8 100644 --- a/integrations/docker/images/chip-cert-bins/Dockerfile +++ b/integrations/docker/images/chip-cert-bins/Dockerfile @@ -251,7 +251,6 @@ RUN case ${TARGETPLATFORM} in \ *) ;; \ esac -RUN npm --prefix third_party/zap/repo/ ci RUN scripts/examples/gn_build_test_example.sh app1 RUN source scripts/activate.sh && scripts/build_python.sh -m platform -d true -i no diff --git a/scripts/helpers/pull_upstream_and_regenerate_zap.sh b/scripts/helpers/pull_upstream_and_regenerate_zap.sh index 434f6c35bbf90a..d3d1a1856dee1f 100755 --- a/scripts/helpers/pull_upstream_and_regenerate_zap.sh +++ b/scripts/helpers/pull_upstream_and_regenerate_zap.sh @@ -2,18 +2,9 @@ set -x git pull upstream master -git submodule update --init --recursive third_party/zap/ git status -cd third_party/zap/repo/ -npm ci -npm run version-stamp -npm rebuild canvas --update-binary -npm run build-spa - -cd ../../../ - scripts/tools/zap_regen_all.py git status diff --git a/scripts/tools/zap/convert.py b/scripts/tools/zap/convert.py index f3b8fc1d1b76fe..0edc09167baa8e 100755 --- a/scripts/tools/zap/convert.py +++ b/scripts/tools/zap/convert.py @@ -92,21 +92,46 @@ def runConversion(zap_file): templates_file = getFilePath('src/app/zap-templates/app-templates.json') zcl_file = detectZclFile(zap_file) - generator_dir = getDirPath('third_party/zap/repo') - os.chdir(generator_dir) - subprocess.check_call(['node', './src-script/zap-convert.js', - '-z', zcl_file, '-g', templates_file, '-o', zap_file, zap_file]) - - -def runBootstrap(): - subprocess.check_call(getFilePath("scripts/tools/zap/zap_bootstrap.sh"), shell=True) + # Accepted environment variables, in order: + # + # ZAP_DEVELOPMENT_PATH - the path to a zap development environment. This is + # a zap checkout, used for local development + # ZAP_INSTALL_PATH - the path where zap-cli exists. This is if zap-cli + # is NOT in the current path + + if 'ZAP_DEVELOPMENT_PATH' in os.environ: + convert_cmd = ['node', 'src-script/zap-start.js', 'convert'] + working_directory = os.environ['ZAP_DEVELOPMENT_PATH'] + elif 'ZAP_INSTALL_PATH' in os.environ: + convert_cmd = [os.path.join(os.environ['ZAP_INSTALL_PATH'], 'zap-cli'), 'convert'] + working_directory = None + else: + convert_cmd = ['zap-cli', 'convert'] + working_directory = None + + try: + subprocess.check_call(convert_cmd + ['-z', zcl_file, '-g', templates_file, '-o', zap_file, zap_file], cwd=working_directory) + except FileNotFoundError as e: + print(f'FAILED TO EXECUTE ZAP CONVERSION: {e.strerror} - "{e.filename}"') + print('*'*80) + print('* You may need to install zap. Please ensure one of these applies:') + print('* - `zap-cli` is in $PATH. Install from https://github.com/project-chip/zap/releases') + print('* see docs/guides/BUILDING.md for details') + print('* - `zap-cli` is in $ZAP_INSTALL_PATH. Use this option if you') + print('* installed zap but do not want to update $PATH') + print('* - Point $ZAP_DEVELOPMENT_PATH to your local copy of zap that you') + print('* develop on (to use a developer build of zap)') + print('*'*80) + sys.exit(1) def main(): checkPythonVersion() zap_file, run_bootstrap = runArgumentsParser() + if run_bootstrap: - runBootstrap() + subprocess.check_call(getFilePath("scripts/tools/zap/zap_bootstrap.sh"), shell=True) + os.chdir(CHIP_ROOT_DIR) runConversion(zap_file) diff --git a/scripts/tools/zap/generate.py b/scripts/tools/zap/generate.py index 0885e83b5ee91e..5c48a41619d1aa 100755 --- a/scripts/tools/zap/generate.py +++ b/scripts/tools/zap/generate.py @@ -149,10 +149,38 @@ def extractGeneratedIdl(output_dir, zap_config_path): def runGeneration(zap_file, zcl_file, templates_file, output_dir): - generator_dir = os.getenv('ZAP_PATH', getDirPath('third_party/zap/repo')) - os.chdir(generator_dir) - subprocess.check_call(['node', './src-script/zap-generate.js', '-z', - zcl_file, '-g', templates_file, '-i', zap_file, '-o', output_dir]) + # Accepted environment variables, in order: + # + # ZAP_DEVELOPMENT_PATH - the path to a zap development environment. This is + # a zap checkout, used for local development + # ZAP_INSTALL_PATH - the path where zap-cli exists. This is if zap-cli + # is NOT in the current path + + if 'ZAP_DEVELOPMENT_PATH' in os.environ: + generate_cmd = ['node', 'src-script/zap-start.js', 'generate'] + working_directory = os.environ['ZAP_DEVELOPMENT_PATH'] + elif 'ZAP_INSTALL_PATH' in os.environ: + generate_cmd = [os.path.join(os.environ['ZAP_INSTALL_PATH'], 'zap-cli'), 'generate'] + working_directory = None + else: + generate_cmd = ['zap-cli', 'generate'] + working_directory = None + + try: + subprocess.check_call(generate_cmd + ['-z', zcl_file, '-g', templates_file, + '-i', zap_file, '-o', output_dir], cwd=working_directory) + except FileNotFoundError as e: + print(f'FAILED TO EXECUTE ZAP GENERATION: {e.strerror} - "{e.filename}"') + print('*'*80) + print('* You may need to install zap. Please ensure one of these applies:') + print('* - `zap-cli` is in $PATH. Install from https://github.com/project-chip/zap/releases') + print('* see docs/guides/BUILDING.md for details') + print('* - `zap-cli` is in $ZAP_INSTALL_PATH. Use this option if you') + print('* installed zap but do not want to update $PATH') + print('* - Point $ZAP_DEVELOPMENT_PATH to your local copy of zap that you') + print('* develop on (to use a developer build of zap)') + print('*'*80) + sys.exit(1) extractGeneratedIdl(output_dir, zap_file) @@ -222,15 +250,13 @@ def runJavaPrettifier(templates_file, output_dir): print('google-java-format error:', err) -def runBootstrap(): - subprocess.check_call(getFilePath("scripts/tools/zap/zap_bootstrap.sh"), shell=True) - - def main(): checkPythonVersion() cmdLineArgs = runArgumentsParser() + if cmdLineArgs.runBootstrap: - runBootstrap() + subprocess.check_call(getFilePath("scripts/tools/zap/zap_bootstrap.sh"), shell=True) + # The maximum memory usage is over 4GB (#15620) os.environ["NODE_OPTIONS"] = "--max-old-space-size=8192" runGeneration(cmdLineArgs.zapFile, cmdLineArgs.zclFile, cmdLineArgs.templateFile, cmdLineArgs.outputDir) diff --git a/scripts/tools/zap/run_zaptool.sh b/scripts/tools/zap/run_zaptool.sh index e2b7ee0a76e680..5e7a1855b4288a 100755 --- a/scripts/tools/zap/run_zaptool.sh +++ b/scripts/tools/zap/run_zaptool.sh @@ -29,15 +29,29 @@ set -e SCRIPT_PATH="$(_get_fullpath "$0")" CHIP_ROOT="${SCRIPT_PATH%/scripts/tools/zap/run_zaptool.sh}" -[[ -n "$1" ]] && ZAP_ARGS=(-i "$(_get_fullpath "$1")") || ZAP_ARGS=() +[[ -n "$1" ]] && ZAP_ARGS="-i \"$(_get_fullpath "$1")\"" || ZAP_ARGS="" -( +if [ ! -z "$ZAP_DEVELOPMENT_PATH" ]; then + WORKING_DIR=$ZAP_DEVELOPMENT_PATH + ZAP_CMD="node src-script/zap-start.js" "$CHIP_ROOT"/scripts/tools/zap/zap_bootstrap.sh +elif [ ! -z "$ZAP_INSTALL_PATH" ]; then + if [[ "$OSTYPE" == "darwin"* ]]; then + ZAP_CMD="$ZAP_INSTALL_PATH/zap.app/Contents/MacOS/zap" + else + ZAP_CMD="$ZAP_INSTALL_PATH/zap" + fi + WORKING_DIR="$CHIP_ROOT" +else + ZAP_CMD="zap" + WORKING_DIR="$CHIP_ROOT" +fi - cd "$CHIP_ROOT/third_party/zap/repo" +( + cd "$WORKING_DIR" - echo "ARGS: ${ZAP_ARGS[@]}" + echo "ARGS: $ZAP_ARGS" if [[ "${ZAP_ARGS[@]}" == *"/all-clusters-app.zap"* ]]; then ZCL_FILE="$CHIP_ROOT/src/app/zap-templates/zcl/zcl-with-test-extensions.json" @@ -45,8 +59,11 @@ CHIP_ROOT="${SCRIPT_PATH%/scripts/tools/zap/run_zaptool.sh}" ZCL_FILE="$CHIP_ROOT/src/app/zap-templates/zcl/zcl.json" fi - node src-script/zap-start.js --logToStdout \ - --gen "$CHIP_ROOT/src/app/zap-templates/app-templates.json" \ - --zcl "$ZCL_FILE" \ - "${ZAP_ARGS[@]}" + bash -c " \ + $ZAP_CMD \ + --logToStdout \ + --gen \"$CHIP_ROOT/src/app/zap-templates/app-templates.json\" \ + --zcl \"$ZCL_FILE\" \ + $ZAP_ARGS \ + " ) diff --git a/scripts/tools/zap/zap_bootstrap.sh b/scripts/tools/zap/zap_bootstrap.sh index b4e459cb3e57b0..56734783f59ed9 100755 --- a/scripts/tools/zap/zap_bootstrap.sh +++ b/scripts/tools/zap/zap_bootstrap.sh @@ -32,14 +32,13 @@ EOF set -e -SCRIPT_PATH="$(_get_fullpath "$0")" -CHIP_ROOT="${SCRIPT_PATH%/scripts/tools/zap/zap_bootstrap.sh}" +if [ -z "$ZAP_DEVELOPMENT_PATH" ]; then + echo 'Please set $ZAP_DEVELOPMENT_PATH to your zap checkout' + exit 1 +fi ( - cd "$CHIP_ROOT" && - git submodule update --init third_party/zap/repo - - cd "third_party/zap/repo" + cd "$ZAP_DEVELOPMENT_PATH" if [ $# -eq 0 ]; then echo "Running ZAP bootstrap" diff --git a/scripts/tools/zap_convert_all.py b/scripts/tools/zap_convert_all.py index 7ffb4454e3965b..15a68cf0ddb1ef 100755 --- a/scripts/tools/zap_convert_all.py +++ b/scripts/tools/zap_convert_all.py @@ -62,15 +62,13 @@ def runArgumentsParser(): return parser.parse_args() -def runBootstrap(): - subprocess.check_call(os.path.join(CHIP_ROOT_DIR, "scripts/tools/zap/zap_bootstrap.sh"), shell=True) - - def main(): args = runArgumentsParser() checkPythonVersion() + if args.run_bootstrap: - runBootstrap() + subprocess.check_call(os.path.join(CHIP_ROOT_DIR, "scripts/tools/zap/zap_bootstrap.sh"), shell=True) + os.chdir(CHIP_ROOT_DIR) targets = getTargets() diff --git a/scripts/tools/zap_regen_all.py b/scripts/tools/zap_regen_all.py index 61c8f6823663eb..18de729f16b3a8 100755 --- a/scripts/tools/zap_regen_all.py +++ b/scripts/tools/zap_regen_all.py @@ -253,10 +253,6 @@ def getTargets(type, test_target): return targets -def runBootstrap(): - subprocess.check_call(os.path.join(CHIP_ROOT_DIR, "scripts/tools/zap/zap_bootstrap.sh"), shell=True) - - def main(): logging.basicConfig( level=logging.INFO, @@ -268,9 +264,11 @@ def main(): targets = getTargets(args.type, args.tests) - if (not args.dry_run): - if (args.run_bootstrap): - runBootstrap() + if not args.dry_run: + + if args.run_bootstrap: + subprocess.check_call(os.path.join(CHIP_ROOT_DIR, "scripts/tools/zap/zap_bootstrap.sh"), shell=True) + for target in targets: target.generate() diff --git a/src/app/zap-templates/zcl/data-model/manufacturers.xml b/src/app/zap-templates/zcl/data-model/manufacturers.xml new file mode 100644 index 00000000000000..d5ceb4a7580fb0 --- /dev/null +++ b/src/app/zap-templates/zcl/data-model/manufacturers.xml @@ -0,0 +1,741 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/app/zap-templates/zcl/zcl-with-test-extensions.json b/src/app/zap-templates/zcl/zcl-with-test-extensions.json index 625c15fb877e21..241753199a7105 100644 --- a/src/app/zap-templates/zcl/zcl-with-test-extensions.json +++ b/src/app/zap-templates/zcl/zcl-with-test-extensions.json @@ -83,7 +83,7 @@ "zll-devices.xml", "zll.xml" ], - "manufacturersXml": "../../../../third_party/zap/repo/zcl-builtin/shared/manufacturers.xml", + "manufacturersXml": "../../../../src/app/zap-templates/zcl/data-model/manufacturers.xml", "options": { "text": { "defaultResponsePolicy": ["Always", "Conditional", "Never"] diff --git a/src/app/zap-templates/zcl/zcl.json b/src/app/zap-templates/zcl/zcl.json index 474ab92b9ac5a9..801b04b57c0c94 100644 --- a/src/app/zap-templates/zcl/zcl.json +++ b/src/app/zap-templates/zcl/zcl.json @@ -77,7 +77,7 @@ "zll-devices.xml", "zll.xml" ], - "manufacturersXml": "../../../../third_party/zap/repo/zcl-builtin/shared/manufacturers.xml", + "manufacturersXml": "../../../../src/app/zap-templates/zcl/data-model/manufacturers.xml", "options": { "text": { "defaultResponsePolicy": ["Always", "Conditional", "Never"] diff --git a/third_party/zap/repo b/third_party/zap/repo deleted file mode 160000 index 22a059d9e97278..00000000000000 --- a/third_party/zap/repo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 22a059d9e97278094cdeb7ec8c8db63d17804e87