diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 582d2e221699..9a6dd664f547 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -10,166 +10,282 @@ trigger: # PR build config is manually overridden in Azure pipelines UI with different secrets pr: none -jobs: - - job: format - dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. - pool: - vmImage: "ubuntu-18.04" - steps: - - task: Cache@2 - inputs: - key: "format | ./WORKSPACE | **/*.bzl" - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true - - - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Run check format scripts" - - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff" - artifactName: format - condition: failed() - - - job: release - displayName: "Linux-x64 release" - dependsOn: ["format"] - # For master builds, continue even if format fails - condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) - timeoutInMinutes: 360 - pool: - vmImage: "ubuntu-18.04" - steps: - - template: bazel.yml - parameters: - ciTarget: bazel.release - - - job: release_arm64 - displayName: "Linux-arm64 release.server_only" - dependsOn: ["format"] - condition: ne(variables['Build.Reason'], 'PullRequest') - pool: "arm-large" - steps: - - template: bazel.yml - parameters: - managedAgent: false - ciTarget: bazel.release.server_only - rbe: false - artifactSuffix: ".arm64" - - - job: bazel - displayName: "Linux-x64" - dependsOn: ["release"] - # For master builds, continue even if format fails - condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) - strategy: - maxParallel: 3 - matrix: - gcc: - CI_TARGET: "bazel.gcc" - clang_tidy: - CI_TARGET: "bazel.clang_tidy" - asan: - CI_TARGET: "bazel.asan" - tsan: - CI_TARGET: "bazel.tsan" - compile_time_options: - CI_TARGET: "bazel.compile_time_options" - timeoutInMinutes: 360 - pool: - vmImage: "ubuntu-18.04" - steps: - - template: bazel.yml - parameters: - ciTarget: $(CI_TARGET) - - - job: coverage - displayName: "Linux-x64 coverage" - dependsOn: ["format"] - timeoutInMinutes: 360 - pool: "x64-large" - steps: - - template: bazel.yml - parameters: - managedAgent: false - ciTarget: bazel.coverage - rbe: false - # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces - bazelBuildExtraOptions: "--test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" - - - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/coverage coverage' - displayName: "Upload Report to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) - condition: always() - - - job: docker - displayName: "Linux-x64 docker" - dependsOn: ["release"] - condition: and(succeeded(), eq(variables['PostSubmit'], 'true'), ne(variables['Build.Reason'], 'PullRequest')) - pool: - vmImage: "ubuntu-18.04" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "bazel.release" - itemPattern: "bazel.release/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - - bash: | - set -e - tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz - ci/docker_ci.sh - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: $(DockerUsername) - DOCKERHUB_PASSWORD: $(DockerPassword) - - - job: macOS - dependsOn: ["format"] - timeoutInMinutes: 360 - pool: - vmImage: "macos-latest" - steps: - - script: ./ci/mac_ci_setup.sh - displayName: "Install dependencies" - - - script: ./ci/mac_ci_steps.sh - displayName: "Run Mac CI" - env: - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/bazel-testlogs/**/test.xml" - testRunTitle: "macOS" - condition: always() - - - job: Windows - dependsOn: ["format"] - timeoutInMinutes: 360 - pool: - vmImage: "windows-latest" - steps: - - bash: ci/run_envoy_docker_windows.sh ci/windows_ci_steps.sh - displayName: "Run Windows CI" - env: - ENVOY_RBE: "true" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) +stages: + - stage: precheck + jobs: + - job: format + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + pool: + vmImage: "ubuntu-18.04" + steps: + - task: Cache@2 + inputs: + key: "format | ./WORKSPACE | **/*.bzl" + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true + + - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Run check format scripts" + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff" + artifactName: format + condition: failed() + + - job: docs + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + pool: + vmImage: "ubuntu-18.04" + steps: + - task: Cache@2 + inputs: + key: "docs | ./WORKSPACE | **/*.bzl" + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true + + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Generate docs" + + - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs' + displayName: "Upload Docs to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.SourcesDirectory)/generated/docs" + artifactName: docs + + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DocsPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DocsPrivateKey)" + condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true)) + + - script: docs/publish.sh + displayName: "Publish to GitHub" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true)) + + - stage: sync + condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true)) + dependsOn: [] + jobs: + - job: filter_example + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(FilterExamplePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(FilterExamplePrivateKey)" + + - bash: ci/filter_example_mirror.sh + displayName: "Sync envoy-filter-example" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + + - job: data_plane_api + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DataPlaneApiPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DataPlaneApiPrivateKey)" + + - bash: ci/api_mirror.sh + displayName: "Sync data-plane-api" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + + - job: go_control_plane + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(GoControlPlanePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(GoControlPlanePrivateKey)" + + - bash: | + cp -a ~/.ssh $(Build.StagingDirectory)/ + ci/run_envoy_docker.sh 'ci/go_mirror.sh' + displayName: "Sync go-control-plane" + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + AZP_BRANCH: $(Build.SourceBranch) + + - stage: linux_x64 + dependsOn: ["precheck"] + # For post-submit builds, continue even if precheck fails + condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true))) + jobs: + - job: release + timeoutInMinutes: 120 + pool: + vmImage: "ubuntu-18.04" + steps: + - template: bazel.yml + parameters: + ciTarget: bazel.release + + - stage: linux_arm64 + dependsOn: ["precheck"] + # For post-submit builds, continue even if precheck fails + condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true))) + jobs: + - job: release + timeoutInMinutes: 120 + pool: "arm-large" + steps: + - template: bazel.yml + parameters: + managedAgent: false + ciTarget: bazel.release.server_only + rbe: false + artifactSuffix: ".arm64" + bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" + + - stage: check + dependsOn: ["linux_x64"] + jobs: + - job: bazel + displayName: "linux_x64" + dependsOn: [] + strategy: + maxParallel: 3 + matrix: + api: + CI_TARGET: "bazel.api" + gcc: + CI_TARGET: "bazel.gcc" + clang_tidy: + CI_TARGET: "bazel.clang_tidy" + asan: + CI_TARGET: "bazel.asan" + tsan: + CI_TARGET: "bazel.tsan" + compile_time_options: + CI_TARGET: "bazel.compile_time_options" + timeoutInMinutes: 120 + pool: + vmImage: "ubuntu-18.04" + steps: + - template: bazel.yml + parameters: + ciTarget: $(CI_TARGET) + + - job: coverage + displayName: "linux_x64" + dependsOn: [] + timeoutInMinutes: 120 + pool: "x64-large" + strategy: + matrix: + coverage: + CI_TARGET: "coverage" + steps: + - template: bazel.yml + parameters: + managedAgent: false + ciTarget: bazel.$(CI_TARGET) + rbe: false + # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces + bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" + + - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)' + displayName: "Upload $(CI_TARGET) Report to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) + condition: always() + + - stage: docker + dependsOn: ["linux_x64"] + jobs: + - job: docker + displayName: "linux multiarch" + pool: + vmImage: "ubuntu-18.04" + steps: + - bash: .azure-pipelines/cleanup.sh + displayName: "Removing tools from agent" + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release" + itemPattern: "bazel.release/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: | + set -e + tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz + ci/docker_ci.sh + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + DOCKERHUB_USERNAME: $(DockerUsername) + DOCKERHUB_PASSWORD: $(DockerPassword) + + - stage: macos + dependsOn: ["precheck"] + jobs: + - job: test + timeoutInMinutes: 180 + pool: + vmImage: "macos-latest" + steps: + - script: ./ci/mac_ci_setup.sh + displayName: "Install dependencies" + + - script: ./ci/mac_ci_steps.sh + displayName: "Run Mac CI" + env: + BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + + - task: PublishTestResults@2 + inputs: + testResultsFiles: "**/bazel-testlogs/**/test.xml" + testRunTitle: "macOS" + condition: always() diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 3bdaab466ba6..000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,71 +0,0 @@ -version: 2.1 - -executors: - ubuntu-build: - description: "A regular build executor based on ubuntu image" - docker: - # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8 - - image: envoyproxy/envoy-build-ubuntu:f21773ab398a879f976936f72c78c9dd3718ca1e - resource_class: xlarge - working_directory: /source - -jobs: - api: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: ci/do_circle_ci.sh bazel.api - - add_ssh_keys: - fingerprints: - - "fb:f3:fe:be:1c:b2:ec:b6:25:f9:7b:a6:87:54:02:8c" - - run: ci/api_mirror.sh - - store_artifacts: - path: /build/envoy/generated - destination: / - - go_control_plane_mirror: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: ci/do_circle_ci.sh bazel.api - - add_ssh_keys: - fingerprints: - - "9d:3b:fe:7c:09:3b:ce:a9:6a:de:de:41:fb:6b:52:62" - - run: ci/go_mirror.sh - - filter_example_mirror: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - add_ssh_keys: - fingerprints: - - "f6:f9:df:90:9c:4b:5f:9c:f4:69:fd:42:94:ff:88:24" - - run: ci/filter_example_mirror.sh - - docs: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: ci/do_circle_ci.sh docs - - add_ssh_keys: - fingerprints: - - "44:c7:a1:9e:f4:9e:a5:33:11:f1:0e:79:e1:55:c9:04" - - run: docs/publish.sh - - store_artifacts: - path: generated/docs - -workflows: - version: 2 - all: - jobs: - - api - - go_control_plane_mirror - - filter_example_mirror - - docs: - filters: - tags: - only: /^v.*/ diff --git a/ci/api_mirror.sh b/ci/api_mirror.sh index 077cdd1d3cfe..03e8ab85d80c 100755 --- a/ci/api_mirror.sh +++ b/ci/api_mirror.sh @@ -3,16 +3,15 @@ set -e CHECKOUT_DIR=../data-plane-api +MAIN_BRANCH="refs/heads/master" +API_MAIN_BRANCH="master" -if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] -then +if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then echo "Cloning..." - git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" + git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" -b "${API_MAIN_BRANCH}" - git -C "$CHECKOUT_DIR" config user.name "data-plane-api(CircleCI)" + git -C "$CHECKOUT_DIR" config user.name "data-plane-api(Azure Pipelines)" git -C "$CHECKOUT_DIR" config user.email data-plane-api@users.noreply.github.com - git -C "$CHECKOUT_DIR" fetch - git -C "$CHECKOUT_DIR" checkout -B master origin/master # Determine last envoyproxy/envoy SHA in envoyproxy/data-plane-api MIRROR_MSG="Mirrored from https://github.com/envoyproxy/envoy" @@ -40,6 +39,6 @@ then done echo "Pushing..." - git -C "$CHECKOUT_DIR" push origin master + git -C "$CHECKOUT_DIR" push origin "${API_MAIN_BRANCH}" echo "Done" fi diff --git a/ci/do_ci.sh b/ci/do_ci.sh index f87e3de67eb5..eb6a038dc5e8 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -246,9 +246,16 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then exit 0 elif [[ "$CI_TARGET" == "bazel.api" ]]; then + # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib, + # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild. + ENVOY_STDLIB="libstdc++" setup_clang_toolchain + export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config echo "Validating API structure..." ./tools/api/validate_structure.py + echo "Testing API and API Boosting..." + bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \ + @envoy_api_canonical//tools:tap2pcap_test @envoy_dev//clang_tools/api_booster/... echo "Building API..." bazel build ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api_canonical//envoy/... echo "Testing API..." @@ -258,7 +265,7 @@ elif [[ "$CI_TARGET" == "bazel.api" ]]; then bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_dev//clang_tools/api_booster/... echo "Testing API boosting (golden C++ tests)..." # We use custom BAZEL_BUILD_OPTIONS here; the API booster isn't capable of working with libc++ yet. - LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config BAZEL_BUILD_OPTIONS="--config=clang" python3.8 ./tools/api_boost/api_boost_test.py + BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" python3.8 ./tools/api_boost/api_boost_test.py exit 0 elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then setup_clang_toolchain diff --git a/ci/filter_example_mirror.sh b/ci/filter_example_mirror.sh index 1d6d5ae05b23..8602b1677e4b 100755 --- a/ci/filter_example_mirror.sh +++ b/ci/filter_example_mirror.sh @@ -4,16 +4,15 @@ set -e ENVOY_SRCDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd) CHECKOUT_DIR=../envoy-filter-example +MAIN_BRANCH="refs/heads/master" +FILTER_EXAMPLE_MAIN_BRANCH="master" -if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] -then +if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then echo "Cloning..." - git clone git@github.com:envoyproxy/envoy-filter-example "$CHECKOUT_DIR" + git clone git@github.com:envoyproxy/envoy-filter-example "$CHECKOUT_DIR" -b "${FILTER_EXAMPLE_MAIN_BRANCH}" - git -C "$CHECKOUT_DIR" config user.name "envoy-filter-example(CircleCI)" + git -C "$CHECKOUT_DIR" config user.name "envoy-filter-example(Azure Pipelines)" git -C "$CHECKOUT_DIR" config user.email envoy-filter-example@users.noreply.github.com - git -C "$CHECKOUT_DIR" fetch - git -C "$CHECKOUT_DIR" checkout -B master origin/master echo "Updating Submodule..." # Update submodule to latest Envoy SHA @@ -26,6 +25,6 @@ then echo "Committing, and Pushing..." git -C "$CHECKOUT_DIR" commit -a -m "Update Envoy submodule to $ENVOY_SHA" - git -C "$CHECKOUT_DIR" push origin master + git -C "$CHECKOUT_DIR" push origin "${FILTER_EXAMPLE_MAIN_BRANCH}" echo "Done" fi diff --git a/ci/go_mirror.sh b/ci/go_mirror.sh index 80be4cc0b532..63f96d0d7969 100755 --- a/ci/go_mirror.sh +++ b/ci/go_mirror.sh @@ -2,7 +2,11 @@ set -e -if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] -then - tools/api/generate_go_protobuf.py +MAIN_BRANCH="refs/heads/master" + +# shellcheck source=ci/setup_cache.sh +. "$(dirname "$0")"/setup_cache.sh + +if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then + BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS}" tools/api/generate_go_protobuf.py fi diff --git a/ci/mac_ci_setup.sh b/ci/mac_ci_setup.sh index b9870f942696..5cd363afa60e 100755 --- a/ci/mac_ci_setup.sh +++ b/ci/mac_ci_setup.sh @@ -20,9 +20,23 @@ function install { fi } -if ! brew update; then - echo "Failed to update homebrew" - exit 1 +function retry () { + local returns=1 i=1 + while ((i<=HOMEBREW_RETRY_ATTEMPTS)); do + if "$@"; then + returns=0 + break + else + sleep "$HOMEBREW_RETRY_INTERVAL"; + ((i++)) + fi + done + return "$returns" +} + +if ! retry brew update; then + # Do not exit early if update fails. + echo "Failed to update homebrew" fi DEPS="automake cmake coreutils go libtool wget ninja" diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 886a2347d378..987dbf48e30b 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -18,6 +18,7 @@ USER_GROUP=root [[ -t 1 ]] && ENVOY_DOCKER_OPTIONS+=" -it" [[ -f .git ]] && [[ ! -d .git ]] && ENVOY_DOCKER_OPTIONS+=" -v $(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)" +[[ -n "${SSH_AUTH_SOCK}" ]] && ENVOY_DOCKER_OPTIONS+="-v ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK} -e SSH_AUTH_SOCK" export ENVOY_BUILD_IMAGE="${IMAGE_NAME}:${IMAGE_ID}" @@ -25,6 +26,7 @@ mkdir -p "${ENVOY_DOCKER_BUILD_DIR}" # Since we specify an explicit hash, docker-run will pull from the remote repo if missing. docker run --rm ${ENVOY_DOCKER_OPTIONS} -e HTTP_PROXY=${http_proxy} -e HTTPS_PROXY=${https_proxy} -e NO_PROXY=${no_proxy} \ -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build -v /var/run/docker.sock:/var/run/docker.sock \ + -e AZP_BRANCH \ -e BAZEL_BUILD_EXTRA_OPTIONS -e BAZEL_EXTRA_TEST_OPTIONS -e BAZEL_REMOTE_CACHE -e ENVOY_STDLIB -e BUILD_REASON \ -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS -e ENVOY_RBE -e FUZZIT_API_KEY -e ENVOY_BUILD_IMAGE \ -e ENVOY_SRCDIR -e ENVOY_BUILD_TARGET -e SYSTEM_PULLREQUEST_TARGETBRANCH -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ diff --git a/docs/build.sh b/docs/build.sh index bc0c302414a1..ab9e4fcad562 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -4,15 +4,20 @@ set -e +RELEASE_TAG_REGEX="^refs/tags/v.*" + +if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then + DOCS_TAG="${AZP_BRANCH/refs\/tags\//}" +fi + # We need to set ENVOY_DOCS_VERSION_STRING and ENVOY_DOCS_RELEASE_LEVEL for Sphinx. # We also validate that the tag and version match at this point if needed. -if [ -n "$CIRCLE_TAG" ] -then +if [[ -n "${DOCS_TAG}" ]]; then # Check the git tag matches the version number in the VERSION file. VERSION_NUMBER=$(cat VERSION) - if [ "v${VERSION_NUMBER}" != "${CIRCLE_TAG}" ]; then + if [[ "v${VERSION_NUMBER}" != "${DOCS_TAG}" ]]; then echo "Given git tag does not match the VERSION file content:" - echo "${CIRCLE_TAG} vs $(cat VERSION)" + echo "${DOCS_TAG} vs $(cat VERSION)" exit 1 fi # Check the version_history.rst contains current release version. @@ -20,9 +25,9 @@ then || (echo "Git tag not found in version_history/current.rst" && exit 1) # Now that we know there is a match, we can use the tag. - export ENVOY_DOCS_VERSION_STRING="tag-$CIRCLE_TAG" + export ENVOY_DOCS_VERSION_STRING="tag-${DOCS_TAG}" export ENVOY_DOCS_RELEASE_LEVEL=tagged - export ENVOY_BLOB_SHA="$CIRCLE_TAG" + export ENVOY_BLOB_SHA="${DOCS_TAG}" else BUILD_SHA=$(git rev-parse HEAD) VERSION_NUM=$(cat VERSION) diff --git a/docs/publish.sh b/docs/publish.sh index b146c42994cb..c56e9bb31066 100755 --- a/docs/publish.sh +++ b/docs/publish.sh @@ -10,35 +10,36 @@ set -e DOCS_DIR=generated/docs -CHECKOUT_DIR=../envoy-docs -BUILD_SHA=`git rev-parse HEAD` - -if [ -n "$CIRCLE_TAG" ] -then - PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy/"$CIRCLE_TAG" -elif [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] -then - PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy/latest +CHECKOUT_DIR=envoy-docs +BUILD_SHA=$(git rev-parse HEAD) + +MAIN_BRANCH="refs/heads/master" +RELEASE_TAG_REGEX="^refs/tags/v.*" + +if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then + PUBLISH_DIR="${CHECKOUT_DIR}"/docs/envoy/"${AZP_BRANCH/refs\/tags\//}" +elif [[ "$AZP_BRANCH" == "${MAIN_BRANCH}" ]]; then + PUBLISH_DIR="${CHECKOUT_DIR}"/docs/envoy/latest else echo "Ignoring docs push" exit 0 fi +DOCS_MAIN_BRANCH="master" + echo 'cloning' -git clone git@github.com:envoyproxy/envoyproxy.github.io "$CHECKOUT_DIR" +git clone git@github.com:envoyproxy/envoyproxy.github.io "${CHECKOUT_DIR}" -b "${DOCS_MAIN_BRANCH}" --depth 1 -git -C "$CHECKOUT_DIR" fetch -git -C "$CHECKOUT_DIR" checkout -B master origin/master rm -fr "$PUBLISH_DIR" mkdir -p "$PUBLISH_DIR" cp -r "$DOCS_DIR"/* "$PUBLISH_DIR" -cd "$CHECKOUT_DIR" +cd "${CHECKOUT_DIR}" -git config user.name "envoy-docs(travis)" +git config user.name "envoy-docs(Azure Pipelines)" git config user.email envoy-docs@users.noreply.github.com -echo 'add' + +set -x + git add . -echo 'commit' git commit -m "docs envoy@$BUILD_SHA" -echo 'push' -git push origin master +git push origin "${DOCS_MAIN_BRANCH}" diff --git a/tools/api/generate_go_protobuf.py b/tools/api/generate_go_protobuf.py index 620e80bb32c5..126a782b9193 100755 --- a/tools/api/generate_go_protobuf.py +++ b/tools/api/generate_go_protobuf.py @@ -4,17 +4,21 @@ from subprocess import check_call import glob import os +import shlex import shutil import sys import re +# Needed for CI to pass down bazel options. +BAZEL_BUILD_OPTIONS = shlex.split(os.environ.get('BAZEL_BUILD_OPTIONS', '')) + TARGETS = '@envoy_api//...' IMPORT_BASE = 'github.com/envoyproxy/go-control-plane' OUTPUT_BASE = 'build_go' REPO_BASE = 'go-control-plane' BRANCH = 'master' MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ ' -USER_NAME = 'go-control-plane(CircleCI)' +USER_NAME = 'go-control-plane(Azure Pipelines)' USER_EMAIL = 'go-control-plane@users.noreply.github.com' @@ -28,7 +32,7 @@ def generateProtobufs(output): # Each rule has the form @envoy_api//foo/bar:baz_go_proto. # First build all the rules to ensure we have the output files. - check_call(['bazel', 'build', '-c', 'fastbuild'] + go_protos) + check_call(['bazel', 'build', '-c', 'fastbuild'] + BAZEL_BUILD_OPTIONS + go_protos) for rule in go_protos: # Example rule: @@ -63,9 +67,7 @@ def git(repo, *args): def cloneGoProtobufs(repo): # Create a local clone of go-control-plane - git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo) - git(repo, 'fetch') - git(repo, 'checkout', '-B', BRANCH, 'origin/master') + git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo, '-b', BRANCH) def findLastSyncSHA(repo): diff --git a/tools/api_boost/api_boost.py b/tools/api_boost/api_boost.py index 8916e8b82822..8e91de120158 100755 --- a/tools/api_boost/api_boost.py +++ b/tools/api_boost/api_boost.py @@ -133,7 +133,6 @@ def ApiBoostTree(target_paths, sp.run([ 'bazel', 'build', - '--config=libc++', '--strip=always', ] + BAZEL_BUILD_OPTIONS + dep_lib_build_targets, check=True)