Reenable Cirque tests (#28350)

* Revert "Disabling Cirque for now"

This reverts commit b587d14.

* Fix cirque test scripts

* Restyled by prettier-json

* Restyled by isort

* Fix ot-br-posix

* Restyled by clang-format

---------

Co-authored-by: Restyled.io <[email protected]>

.github/workflows/cirque.yaml

@@ -0,0 +1,130 @@
+# Copyright (c) 2020-2023 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Cirque
+
+on:
+    push:
+    pull_request:
+    merge_group:
+    workflow_dispatch:
+
+concurrency:
+    group: ${{ github.ref }}-${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.event.number) || (github.event_name == 'workflow_dispatch' && github.run_number) || github.sha }}
+    cancel-in-progress: true
+
+env:
+    CHIP_NO_LOG_TIMESTAMPS: true
+
+jobs:
+    cirque:
+        name: Cirque
+
+        env:
+            GITHUB_CACHE_PATH: /tmp/cirque-cache
+
+        runs-on: ubuntu-latest
+        if: github.actor != 'restyled-io[bot]'
+
+        # need to run with privilege, which isn't supported by job.XXX.contaner
+        #  https://github.com/actions/container-action/issues/2
+        #         container:
+        #             image: ghcr.io/project-chip/chip-build-cirque:0.8
+        #             volumes:
+        #                 - "/tmp:/tmp"
+        #                 - "/dev/pts:/dev/pts"
+        #             env:
+        #                 LOG_DIR: /tmp/cirque_test_output/
+        #                 CLEANUP_DOCKER_FOR_CI: 1
+        #             options: "--privileged"
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+            - name: Checkout submodules
+              uses: ./.github/actions/checkout-submodules
+              with:
+                platform: linux
+
+            - name: Bootstrap Cache
+              uses: ./.github/actions/bootstrap-cache
+            - name: Bootstrap Cirque
+              run: |
+                  integrations/docker/images/stage-2/chip-build-cirque/run.sh \
+                    -- sh -c " \
+                       git config --global --add safe.directory '*' \
+                       && bash scripts/bootstrap.sh \
+                       && chown -R $(id -u):$(id -g) .environment \
+                    "
+
+            - name: Get Cirque Bootstrap cache key
+              id: cirque-bootstrap-cache-key
+              run: echo "val=$(scripts/tests/cirque_tests.sh cachekeyhash)" >> $GITHUB_OUTPUT
+            - uses: Wandalen/[email protected]
+              name: Cirque Bootstrap cache
+              if: ${{ !env.ACT }}
+              continue-on-error: true
+              timeout-minutes: 10
+              with:
+                action: buildjet/cache@v3
+                attempt_limit: 3
+                attempt_delay: 2000
+                with: |
+                  key: ${{ runner.os }}-cirque-${{ steps.cirque-bootstrap-cache-key.outputs.val }}
+                  restore-keys: ${{ runner.os }}-cirque-
+                  path: ${{ env.GITHUB_CACHE_PATH }}
+            - name: Cirque Bootstrap
+              run: |
+                  integrations/docker/images/stage-2/chip-build-cirque/run.sh \
+                    --env GITHUB_ACTION_RUN=1 \
+                    --env GITHUB_CACHE_PATH=${{ env.GITHUB_CACHE_PATH }} \
+                    --volume /tmp:/tmp \
+                    -- scripts/tests/cirque_tests.sh bootstrap
+
+            - name: Artifact suffix
+              id: outsuffix
+              uses: haya14busa/action-cond@v1
+              if: ${{ !env.ACT }}
+              with:
+                  cond: ${{ github.event.pull_request.number == '' }}
+                  if_true: "${{ github.sha }}"
+                  if_false: "pull-${{ github.event.pull_request.number }}"
+            - name: Build Binaries
+              run: |
+                  integrations/docker/images/stage-2/chip-build-cirque/run.sh \
+                    -- sh -c " \
+                       git config --global --add safe.directory '*' \
+                       && scripts/build/gn_gen_cirque.sh \
+                      "
+            - name: Run Tests
+              run: |
+                  integrations/docker/images/stage-2/chip-build-cirque/run.sh \
+                     --env LOG_DIR=/tmp/cirque_test_output \
+                     --env CLEANUP_DOCKER_FOR_CI=1 \
+                     --env GITHUB_ACTION_RUN=1 \
+                     --privileged \
+                     --volume /tmp:/tmp \
+                     --volume /dev/pts:/dev/pts \
+                     -- sh -c " \
+                       git config --global --add safe.directory '*' \
+                       && scripts/tests/cirque_tests.sh run_all_tests \
+                      "
+                   
+
+            - name: Uploading Binaries
+              uses: actions/upload-artifact@v3
+              if: ${{ always() && !env.ACT }}
+              with:
+                  name: cirque_log-${{steps.outsuffix.outputs.value}}-logs
+                  path: /tmp/cirque_test_output/

integrations/docker/images/stage-2/chip-build-cirque/Dockerfile

@@ -0,0 +1,46 @@
+ARG VERSION=latest
+FROM ghcr.io/project-chip/chip-build:${VERSION}
+LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip
+
+# Bazel
+RUN set -x \
+   && apt-get update \
+   && DEBIAN_FRONTEND=noninteractive apt-get install -fy \
+   curl gnupg \
+   && curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor > bazel.gpg \
+   && mv bazel.gpg /etc/apt/trusted.gpg.d/ \
+   && echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list \
+   && apt-get update \
+   && DEBIAN_FRONTEND=noninteractive apt-get install -fy \
+   bazel \
+   && : # aids diffs
+
+# Docker
+RUN set -x \
+   && apt-get update \
+   && DEBIAN_FRONTEND=noninteractive apt-get install -fy \
+   curl gnupg-agent apt-transport-https ca-certificates \
+   software-properties-common \
+   && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+   && python3.8 `which add-apt-repository` \
+   "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
+   && apt-get update \
+   && DEBIAN_FRONTEND=noninteractive apt-get install -fy \
+   docker-ce docker-ce-cli containerd.io \
+   && : # aids diffs
+
+# Other Cirque prereqs
+RUN set -x \
+   && apt-get update \
+   && DEBIAN_FRONTEND=noninteractive apt-get install -fy \
+   sudo socat psmisc tigervnc-standalone-server xorg xauth \
+   python3-pip python3-venv libdbus-glib-1-dev \
+   uuid-runtime libgirepository1.0-dev \
+   && : # aids diffs
+
+COPY requirements_nogrpc.txt /requirements.txt
+
+RUN set -x \
+   && pip3 install -r requirements.txt \
+   && xinit -- /usr/bin/Xvnc \
+   && : # aids diffs

integrations/docker/images/stage-2/chip-build-cirque/README.md

@@ -0,0 +1,36 @@
+# Docker connectedhomeip:chip-build-cirque
+
+project-chip/chip-build-cirque is the name of the Docker image used by CHIP for
+continuous integration and other builds when using Cirque. It builds upon the
+chip-build image and adds system dependencies needed by Cirque.
+
+Contents of this directory:
+
+-   build.sh - utility for building (and optionally) tagging and pushing the
+    chip-build Docker image
+-   version - the semver-style version of the image in use for this branch of
+    CHIP
+-   Dockerfile - description of the image
+
+Please update version when any required tooling is updated. Some rough
+guidelines:
+
+-   Updating a tool? Increment dot version unless the tool has a major version
+    delta or a backward incompatibility
+-   Adding a tool? Increment minor version: e.g. 0.2.1 -> 0.3.0
+-   Removing a tool? Increment major version: e.g. 1.2 -> 2.0
+
+Note, you must have privileged access to the connectedhomeip area on Docker Hub
+to push the image.
+
+Typical use:
+
+1.  new build tool dependency identified
+2.  add tool to Dockerfile
+3.  update version
+4.  `$ ./build.sh`, which installs the image locally as the new version
+5.  update the image version in the devcontainer.json and CI manifests
+6.  verify that the build works locally in the new image
+7.  `$ docker login`
+8.  `$ ./build.sh --push --latest`, _*Note:*_ omit `--latest` unless on the
+    master branch

integrations/docker/images/stage-2/chip-build-cirque/build.sh

@@ -0,0 +1 @@
+../../../build.sh

integrations/docker/images/stage-2/chip-build-cirque/requirements_nogrpc.txt

@@ -0,0 +1,8 @@
+cmd2
+docker >= 4.1.0
+flask == 2.2.2
+pycodestyle >= 2.5.0
+pylint == 2.4
+pyroute2 >= 0.5.7
+six >= 1.12
+toml

integrations/docker/images/stage-2/chip-build-cirque/run.sh

@@ -0,0 +1 @@
+../../../run.sh

integrations/docker/images/stage-2/chip-build-cirque/version

@@ -0,0 +1 @@
+../../base/chip-build/version

integrations/docker/images/stage-2/chip-cirque-device-base/CHIPCirqueDaemon.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2020 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from enum import Enum
+from multiprocessing.connection import Client, Listener
+from pathlib import Path
+
+import click
+
+log = logging.getLogger()
+log.setLevel(logging.DEBUG)
+
+sh = logging.StreamHandler()
+sh.setFormatter(
+    logging.Formatter(
+        '%(asctime)s [%(name)s] %(levelname)s %(message)s'))
+log.addHandler(sh)
+
+
+class CommandStatus(Enum):
+    """Enum class for passing status code of execute CirqueDaemon command, not CHIP error codes."""
+    SUCCESS = 0
+    FAILURE = 1
+    UNKNOWN_COMMAND = 2
+    INVALID_ARGUMENT = 3
+
+
+@dataclass
+class CommandResponse:
+    """Class for holding status of running CirqueDaemon commands."""
+    status_code: CommandStatus
+    error_message: str = ""
+
+
+class ShellCommand:
+    def __init__(self, args=None):
+        self._args = args
+
+    def __call__(self):
+        if not self._args:
+            return CommandResponse(CommandStatus.INVALID_ARGUMENT, "Cannot spwan background process")
+        log.info("Will run command: {}".format(self._args))
+        try:
+            # As the command will be execued in background, we won't return the exit code of the program.
+            subprocess.Popen(
+                self._args, stdout=sys.stdout, stderr=sys.stderr)
+            return CommandResponse(CommandStatus.SUCCESS)
+        except Exception as ex:
+            return CommandResponse(CommandStatus.FAILURE, "Failed to run command: {}".format(ex))
+
+
+class InvalidCommand:
+    def __init__(self, args=None):
+        self._args = args
+
+    def __call__(self):
+        return CommandResponse(CommandStatus.FAILURE, "invalid command")
+
+
+SERVER_ADDRESS = "/tmp/cirque-helper.socket"
+CLIENT_WAIT_TIMEOUT_SECONDS = 5
+
+
+def CommandFactory(args):
+    commands = {
+        "run": ShellCommand
+    }
+    if len(args) == 0:
+        return InvalidCommand()
+    return commands.get(args[0], InvalidCommand)(args[1:])
+
+
+def ServerMain(args):
+    extraOptions = {
+        "otbr-agent": ShellCommand(["otbr-agent", "-I", "wpan0", "-B", "eth0", "spinel+hdlc+uart:///dev/ttyUSB0"])
+    }
+
+    with Listener(SERVER_ADDRESS) as listener:
+        log.info("Server running on {}".format(SERVER_ADDRESS))
+        for extraOption in args:
+            cmd = extraOptions.get(extraOption, InvalidCommand())
+            cmd()
+
+        while True:
+            with listener.accept() as conn:
+                log.info("Received connection")
+                cmd = CommandFactory(conn.recv())
+                conn.send(cmd())
+
+
+def ClientMain(args):
+    if len(args) == 0:
+        sys.exit(1)
+    # The server may start very slowly, wait for a few seconds to see if the server will start.
+    for _ in range(CLIENT_WAIT_TIMEOUT_SECONDS):
+        socks = Path(SERVER_ADDRESS)
+        if socks.exists():
+            break
+        time.sleep(1)
+    # If the address does not exist, Client constructor will throw an exception, so no need to add a flag.
+    with Client(SERVER_ADDRESS) as conn:
+        conn.send(args)
+        res = conn.recv()
+        print(res)
+        if res.status_code != CommandStatus.SUCCESS:
+            sys.exit(1)
+
+
+@click.command()
+@click.option('--server', is_flag=True)
+@click.argument('command', nargs=-1)
+def main(server, command):
+    if server:
+        ServerMain(command)
+    else:
+        ClientMain(command)
+
+
+if __name__ == '__main__':
+    main()