Initial pybind11 Controller Wrapper (#8428)

CHIP_ERROR update from Kevin

Fixes small errors in create_error_wrapper based off the new CHIPError class

Fixing darin workflow to install python3.9

Adding full path to gitmodules

.github/workflows/build.yaml

@@ -166,8 +166,82 @@ jobs:
             - name: Perform CodeQL Analysis
               if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/master' }}
               uses: github/codeql-action/analyze@v1
+    build_linux_python_lib:
+        name: Build on Linux (python_lib)
+        timeout-minutes: 60
+
+        runs-on: ubuntu-latest
+        if: github.actor != 'restyled-io[bot]'
+
+        container:
+            image: connectedhomeip/chip-build:0.5.0
+            volumes:
+                - "/tmp/log_output:/tmp/test_logs"
+            options:
+                --sysctl "net.ipv6.conf.all.disable_ipv6=0
+                net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1"
+
+        steps:
+            - name: Dump GitHub context
+              env:
+                GITHUB_CONTEXT: ${{ toJSON(github) }}
+              run: echo "$GITHUB_CONTEXT"
+            - name: Dump Concurrency context
+              env:
+                CONCURRENCY_CONTEXT: ${{ github.ref }}-${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.event.number) || (github.event_name == 'workflow_dispatch' && github.run_number) || github.sha }}
+              run: echo "$CONCURRENCY_CONTEXT"
+            - name: Checkout
+              uses: actions/checkout@v2
+              with:
+                  submodules: true
+            - name: Initialize CodeQL
+              if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/master' }}
+              uses: github/codeql-action/init@v1
+              with:
+                  languages: "cpp"
+
+            - name: Bootstrap
+              timeout-minutes: 10
+              run: scripts/build/gn_bootstrap.sh
+            # - name: Uploading bootstrap logs
+            #   uses: actions/upload-artifact@v2
+            #   if: ${{ always() }}
+            #   with:
+            #       name: bootstrap-logs
+            #       path: |
+            #        .environment/gn_out/.ninja_log
+            #        .environment/pigweed-venv/*.log
+            - name: Setup Build, Run Build and Run Tests
+              timeout-minutes: 50
+              run: |
+                  for BUILD_TYPE  in python_lib; do
+                      case $BUILD_TYPE in
+                          "python_lib") GN_ARGS='enable_rtti=true enable_pylib=true chip_config_memory_debug_checks=false chip_config_memory_debug_dmalloc=false';;
+                      esac
+
+                      scripts/build/gn_gen.sh --args="$GN_ARGS"
+                      scripts/build/gn_build.sh
+                      scripts/tests/gn_tests.sh
+                  done
+            # TODO Log Upload https://github.com/project-chip/connectedhomeip/issues/2227
+            # TODO https://github.com/project-chip/connectedhomeip/issues/1512
+            # - name: Run Code Coverage
+            #   if: ${{ contains('main', env.BUILD_TYPE) }}
+            #   run: scripts/tools/codecoverage.sh
+            # - name: Upload Code Coverage
+            #   if: ${{ contains('main', env.BUILD_TYPE) }}
+            #   run: bash <(curl -s https://codecov.io/bash)
+            - name: Remove third_party binaries for CodeQL Analysis
+              run: find out -type d -name "third_party" -exec rm -rf {} +
+            - name: Remove dbus binaries for CodeQL Analysis
+              run: find out -type d -name "dbus" -exec rm -rf {} +
+            # - name: Remove nrfxlib binaries for CodeQL Analysis
+            #   run: find . -type d -name "nrfxlib" -exec rm -rf {} +
+            - name: Perform CodeQL Analysis
+              if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/master' }}
+              uses: github/codeql-action/analyze@v1
     build_darwin:
-        name: Build on Darwin
+        name: Build on Darwin (clang, python_lib)
         timeout-minutes: 60
 
         env:
@@ -202,19 +276,22 @@ jobs:
                   path: |
                    .environment/gn_out/.ninja_log
                    .environment/pigweed-venv/*.log
-            - name: Setup Build
+            - name: Setup Build, Run Build and Run Tests
+              timeout-minutes: 50
               # Just go ahead and do the "all" build; on Darwin that's fairly
               # fast.  If this ever becomes slow, we can think about ways to do
               # the examples-linux-standalone.yaml tests on darwin without too
               # much code duplication.
               run: |
-                  scripts/build/gn_gen.sh --args='is_clang=true target_os="all"'
-            - name: Run Build
-              timeout-minutes: 30
-              run: scripts/build/gn_build.sh
-            - name: Run Tests
-              timeout-minutes: 10
-              run: scripts/tests/gn_tests.sh
+                  for BUILD_TYPE  in clang python_lib; do
+                      case $BUILD_TYPE in
+                          "clang") GN_ARGS='is_clang=true target_os="all"';;
+                          "python_lib") GN_ARGS='enable_rtti=true enable_pylib=true';;
+                      esac
+                      scripts/build/gn_gen.sh --args="$GN_ARGS"
+                      scripts/build/gn_build.sh
+                      scripts/tests/gn_tests.sh
+                  done
             # TODO Log Upload https://github.com/project-chip/connectedhomeip/issues/2227
             # TODO https://github.com/project-chip/connectedhomeip/issues/1512
             # - name: Run Code Coverage

.github/workflows/darwin.yaml

@@ -37,7 +37,7 @@ jobs:
               with:
                   submodules: true
             - name: Setup Environment
-              run: brew install openssl pkg-config
+              run: brew install openssl pkg-config [email protected]
             - name: Fix pkgconfig link
               working-directory: /usr/local/lib/pkgconfig
               run: |

.gitmodules

@@ -1,79 +1,79 @@
 [submodule "nlassert"]
 	path = third_party/nlassert/repo
 	url = https://github.com/nestlabs/nlassert.git
-	branch = master
+	branch  = master
 [submodule "nlfaultinjection"]
 	path = third_party/nlfaultinjection/repo
 	url = https://github.com/nestlabs/nlfaultinjection.git
-	branch = master
+	branch  = master
 [submodule "nlio"]
 	path = third_party/nlio/repo
 	url = https://github.com/nestlabs/nlio.git
-	branch = master
+	branch  = master
 [submodule "nlunit-test"]
 	path = third_party/nlunit-test/repo
 	url = https://github.com/nestlabs/nlunit-test.git
-	branch = master
+	branch  = master
 [submodule "mbedtls"]
 	path = third_party/mbedtls/repo
 	url = https://github.com/ARMmbed/mbedtls.git
-	branch = mbedtls-2.25
+	branch  = mbedtls-2.25
 [submodule "qrcode"]
 	path = examples/common/QRCode/repo
 	url = https://github.com/nayuki/QR-Code-generator.git
-	branch = master
+	branch  = master
 [submodule "m5stack-tft"]
 	path = examples/common/m5stack-tft/repo
 	url = https://github.com/jeremyjh/ESP32_TFT_library.git
-	branch = master
+	branch  = master
 [submodule "pigweed"]
 	path = third_party/pigweed/repo
 	url = https://github.com/google/pigweed.git
-	branch = main
+	branch  = main
 [submodule "openthread"]
 	path = third_party/openthread/repo
 	url = https://github.com/openthread/openthread.git
-	branch = main
+	branch  = main
 [submodule "ot-br-posix"]
 	path = third_party/ot-br-posix/repo
 	url = https://github.com/openthread/ot-br-posix.git
-	branch = main
+	branch  = main
 [submodule "bluez"]
 	path = third_party/bluez/repo
 	url = https://github.com/bluez/bluez.git
-	branch = master
+	branch  = master
 [submodule "cirque"]
 	path = third_party/cirque/repo
 	url = https://github.com/openweave/cirque.git
-	branch = master
+	branch  = master
 [submodule "happy"]
 	path = third_party/happy/repo
 	url = https://github.com/openweave/happy.git
-	branch = master
+	branch  = master
 [submodule "nanopb"]
 	path = third_party/nanopb/repo
 	url = https://github.com/nanopb/nanopb.git
-	branch = master
+	branch  = master
 [submodule "qpg_sdk"]
 	path = third_party/qpg_sdk/repo
 	url = https://github.com/Qorvo/qpg-connectedhomeip
-	branch = master
+	branch  = master
 [submodule "zap"]
 	path = third_party/zap/repo
 	url = https://github.com/project-chip/zap.git
-	branch = master
+	branch  = master
 [submodule "freertos"]
 	path = third_party/freertos/repo
 	url = https://github.com/FreeRTOS/FreeRTOS-Kernel.git
-	branch = V10.3.1-kernel-only
+	branch  = V10.3.1-kernel-only
 [submodule "efr32_sdk"]
 	path = third_party/efr32_sdk/repo
 	url = https://github.com/SiliconLabs/sdk_support.git
-	branch = master
+	branch  = master
 [submodule "simw-top-mini"]
 	path = third_party/simw-top-mini/repo
 	url = https://github.com/NXP/plug-and-trust.git
-	branch = int/CHIPSE_Release
+	branch  = int/CHIPSE_Release
 [submodule "third_party/openthread/ot-nxp"]
 	path = third_party/openthread/ot-nxp
 	url = https://github.com/openthread/ot-nxp.git
@@ -86,22 +86,22 @@
 [submodule "third_party/mbed-os/repo"]
 	path = third_party/mbed-os/repo
 	url = https://github.com/ARMmbed/mbed-os.git
-	branch = feature-chip
+	branch  = feature-chip
 [submodule "third_party/wifi-ism43362/repo"]
 	path = third_party/wifi-ism43362/repo
 	url = https://github.com/ATmobica/wifi-ism43362.git
-	branch = master
+	branch  = master
 [submodule "third_party/mbed-os-posix-socket/repo"]
 	path = third_party/mbed-os-posix-socket/repo
 	url = https://github.com/ARMmbed/mbed-os-posix-socket.git
-	branch = main
+	branch  = main
 [submodule "p6/abstraction-rtos"]
 	path = third_party/p6/p6_sdk/libs/abstraction-rtos
 	url = https://github.com/Infineon/abstraction-rtos
 [submodule "p6/bluetooth-freertos"]
 	path = third_party/p6/p6_sdk/libs/bluetooth-freertos
 	url = https://github.com/Infineon/bluetooth-freertos
-	branch = v2.X
+	branch  = v2.X
 [submodule "p6/btstack"]
 	path = third_party/p6/p6_sdk/libs/btstack
 	url = https://github.com/Infineon/btstack
@@ -159,3 +159,7 @@
 [submodule "p6/recipe-make-cat1a"]
 	path = third_party/p6/p6_sdk/libs/recipe-make-cat1a
 	url = https://github.com/Infineon/recipe-make-cat1a
+[submodule "third_party/pybind11/repo"]
+	path = third_party/pybind11/repo
+	url = https://github.com/pybind/pybind11
+	branch  = stable

BUILD.gn

@@ -37,6 +37,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
     chip_enable_python_modules =
         (current_os == "mac" || current_os == "linux") &&
         (host_cpu == "x64" || host_cpu == "arm64")
+    enable_pylib = false
   }
 
   # Python packages for supporting specific targets.
@@ -112,6 +113,9 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
         deps += [ "${chip_root}/src/tools/chip-cert" ]
       }
       if (chip_enable_python_modules) {
+        if (enable_pylib) {
+          deps += [ "${chip_root}/src/pybindings/pycontroller" ]
+        }
         deps += [ "${chip_root}/src/controller/python" ]
       }
     }
@@ -129,6 +133,9 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
       data_deps = [ "${chip_root}/examples/chip-tool" ]
 
       if (chip_enable_python_modules) {
+        if (enable_pylib) {
+          data_deps += [ "${chip_root}/src/pybindings/pycontroller" ]
+        }
         data_deps += [ "${chip_root}/src/controller/python" ]
       }
 

build/config/compiler/BUILD.gn

@@ -356,6 +356,11 @@ config("sanitize_default") {
 config("fuzzing_default") {
 }
 
+declare_args() {
+  # Enable Runtime Type Information (RTTI)
+  enable_rtti = false
+}
+
 config("no_rtti") {
   cflags_cc = [ "-fno-rtti" ]
 }
@@ -365,7 +370,11 @@ config("rtti") {
 }
 
 config("rtti_default") {
-  configs = [ ":no_rtti" ]
+  if (enable_rtti) {
+    configs = [ ":rtti" ]
+  } else {
+    configs = [ ":no_rtti" ]
+  }
 }
 
 config("no_exceptions") {

integrations/docker/images/chip-build/Dockerfile

@@ -50,10 +50,9 @@ RUN set -x \
     ninja-build \
     openjdk-8-jdk \
     pkg-config \
-    python3 \
-    python3-dev \
-    python3-pip \
-    python3-venv \
+    python3.9 \
+    python3.9-dev \
+    python3.9-venv \
     rsync \
     shellcheck \
     strace \
@@ -74,14 +73,15 @@ RUN set -x \
     && exec bash \
     && : # last line
 
-# Python 2 and PIP
+# Python 3.9 and PIP
 RUN set -x \
-    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive  apt-get update \
+    && DEBIAN_FRONTEND=noninteractive  apt-get install -y libgirepository1.0-dev \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common \
     && add-apt-repository universe \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y python python2 \
-    && curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \
-    && python2 get-pip.py \
+    && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
+    && python3.9 get-pip.py \
+    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 \
     && rm -rf /var/lib/apt/lists/ \
     && : # last line
 

integrations/docker/images/chip-build/version

@@ -1 +1 @@
-0.4.37
+0.5.0