From 02011b3e41b33e69c7242a1a3c57daf4da7b77b7 Mon Sep 17 00:00:00 2001 From: Ryan Kuester Date: Thu, 21 Nov 2024 16:50:14 -0600 Subject: [PATCH] ci(bazel): add test jobs for compression (#2924) Add GitHub CI jobs that run `bazel test` with compression enabled. Add as new jobs, as opposed to additional steps within the existing jobs, so they can run in parallel. Similar additions for the Make build system will follow in future commits. BUG=#2636 --- .github/workflows/ci.yml | 71 +++++++++++++++++++ .../ci_build/test_bazel_with_compression.sh | 26 +++++++ .../test_bazel_with_compression_asan.sh | 29 ++++++++ .../test_bazel_with_compression_msan.sh | 29 ++++++++ 4 files changed, 155 insertions(+) create mode 100755 tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression.sh create mode 100755 tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_asan.sh create mode 100755 tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_msan.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6491c509de..358d18eee3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,26 @@ jobs: run: | tensorflow/lite/micro/tools/ci_build/test_bazel.sh + bazel_tests_with_compression: + runs-on: ubuntu-latest + + name: Bazel with Compression (presubmit) + steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.trigger-sha }} + - name: Install dependencies + run: | + sudo ci/install_bazelisk.sh + pip3 install Pillow + pip3 install numpy + - name: Test + run: | + tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression.sh + bazel_msan: runs-on: ubuntu-latest @@ -65,6 +85,31 @@ jobs: run: | tensorflow/lite/micro/tools/ci_build/test_bazel_msan.sh + bazel_with_compression_msan: + runs-on: ubuntu-latest + + name: Bazel with Compression [msan] (presubmit) + steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.trigger-sha }} + - name: Install dependencies + run: | + sudo ci/install_bazelisk.sh + pip3 install Pillow + pip3 install numpy + - name: Fix kernel mmap rnd bits + # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with + # high-entropy ASLR in much newer kernels that GitHub runners are + # using leading to random crashes: https://reviews.llvm.org/D148280 + run: sudo sysctl vm.mmap_rnd_bits=28 + - name: Test + run: | + tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_msan.sh + bazel_asan: runs-on: ubuntu-latest @@ -90,6 +135,32 @@ jobs: run: | tensorflow/lite/micro/tools/ci_build/test_bazel_asan.sh + bazel_with_compression_asan: + runs-on: ubuntu-latest + + name: Bazel with Compression [asan] (presubmit) + steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.trigger-sha }} + - name: Install dependencies + run: | + sudo ci/install_bazelisk.sh + pip3 install Pillow + pip3 install numpy + - name: Fix kernel mmap rnd bits + # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with + # high-entropy ASLR in much newer kernels that GitHub runners are + # using leading to random crashes: https://reviews.llvm.org/D148280 + run: sudo sysctl vm.mmap_rnd_bits=28 + - name: Test + run: | + tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_asan.sh + + # TODO(b/178621680): enable ubsan once bazel + clang + ubsan errors are fixed. # bazel_ubsan: # runs-on: ubuntu-latest diff --git a/tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression.sh b/tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression.sh new file mode 100755 index 00000000000..2c191329d05 --- /dev/null +++ b/tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2024 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR=${SCRIPT_DIR}/../../../../.. +cd "${ROOT_DIR}" + +bazel test //... \ + --//:with_compression \ + --config=ci diff --git a/tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_asan.sh b/tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_asan.sh new file mode 100755 index 00000000000..a7c169aa580 --- /dev/null +++ b/tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_asan.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2024 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR=${SCRIPT_DIR}/../../../../.. +cd "${ROOT_DIR}" + +bazel test //... \ + --//:with_compression \ + --config=ci \ + --config=asan \ + --build_tag_filters=-noasan \ + --test_tag_filters=-noasan diff --git a/tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_msan.sh b/tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_msan.sh new file mode 100755 index 00000000000..205ec33bde4 --- /dev/null +++ b/tensorflow/lite/micro/tools/ci_build/test_bazel_with_compression_msan.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2024 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR=${SCRIPT_DIR}/../../../../.. +cd "${ROOT_DIR}" + +bazel test //... \ + --//:with_compression \ + --config=ci \ + --config=msan \ + --build_tag_filters=-nomsan \ + --test_tag_filters=-nomsan