From c0c67e77040892e4d6e61935653431d249775cca Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 10:26:03 -0400 Subject: [PATCH 01/11] unify management of MSRV This PR standardizes where all crates set the MSRV (azure_core), as well as updates the actions to build on a matrix of 'stable', 'nightly', and the rust-version set in `azure_core`. This allows developers of the SDK to run each of the build tasks exactly as they are run in CICD with minimal configuration. --- .github/workflows/build.yml | 61 ++++++++++++++++++++++++++------ eng/scripts/emulator_tests.sh | 12 +++++++ sdk/data_cosmos/Cargo.toml | 1 - sdk/data_tables/Cargo.toml | 1 - sdk/iot_hub/Cargo.toml | 1 - sdk/security_keyvault/Cargo.toml | 1 - sdk/storage/Cargo.toml | 1 - sdk/storage_blobs/Cargo.toml | 1 - sdk/storage_datalake/Cargo.toml | 1 - sdk/storage_queues/Cargo.toml | 1 - 10 files changed, 62 insertions(+), 19 deletions(-) create mode 100755 eng/scripts/emulator_tests.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 978caea7fd..1dbdeb31e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,22 +9,44 @@ env: CARGO_INCREMENTAL: 0 jobs: + msrv: + name: Get minimum supported rust version + runs-on: ubuntu-20.04 + outputs: + msrv: ${{ steps.msrv.outputs.msrv }} + steps: + - uses: actions/checkout@v3 + - run: | + MSRV=$(cargo metadata --format-version=1 --no-deps | jq '.packages[] | select(.name == "azure_core").rust_version' -r) + echo ::set-output name=msrv::${MSRV} + code-style: name: Code Style runs-on: ubuntu-20.04 + needs: + - msrv + strategy: + matrix: + # we don't care about code style from nightly + build: + - stable + - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 - - run: eng/scripts/code_style.sh + - run: eng/scripts/code_style.sh ${{ matrix.build }} test-sdk: name: SDK Tests runs-on: ubuntu-20.04 + needs: + - msrv strategy: matrix: build: - stable - nightly + - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 @@ -33,10 +55,18 @@ jobs: test-services: name: Services Tests runs-on: ubuntu-20.04 + needs: + - msrv + strategy: + matrix: + build: + - stable + - nightly + - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 - - run: eng/scripts/services_tests.sh + - run: eng/scripts/services_tests.sh ${{ matrix.build }} - name: display free disk space run: df -h / if: ${{ always() }} @@ -44,17 +74,18 @@ jobs: test-integration: name: Integration Tests runs-on: ubuntu-20.04 + needs: + - msrv + strategy: + matrix: + build: + - stable + - nightly + - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - - run: rustup update --no-self-update stable - uses: Swatinem/rust-cache@v2 - - - name: emulator integration tests - run: | - npm install azurite@3.13.1 - npx azurite & - cargo test --features test_integration - + - run: eng/scripts/emulator_tests.sh ${{ matrix.build }} - name: display free disk space run: df -h / if: ${{ always() }} @@ -62,10 +93,18 @@ jobs: test-e2e: name: E2E Tests runs-on: ubuntu-20.04 + needs: + - msrv + strategy: + matrix: + build: + - stable + - nightly + - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 - - run: eng/scripts/e2e_tests.sh + - run: eng/scripts/e2e_tests.sh ${{ matrix.build }}} - name: display free disk space run: df -h / if: ${{ always() }} diff --git a/eng/scripts/emulator_tests.sh b/eng/scripts/emulator_tests.sh new file mode 100755 index 0000000000..082a04b145 --- /dev/null +++ b/eng/scripts/emulator_tests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -eux -o pipefail +cd $(dirname ${BASH_SOURCE[0]})/../../ + +BUILD=${1:-stable} + +npm install azurite@3.13.1 +npx azurite & + +rustup update --no-self-update ${BUILD} +cargo +${BUILD} test --features test_integration diff --git a/sdk/data_cosmos/Cargo.toml b/sdk/data_cosmos/Cargo.toml index af99132f09..b46509769b 100644 --- a/sdk/data_cosmos/Cargo.toml +++ b/sdk/data_cosmos/Cargo.toml @@ -11,7 +11,6 @@ documentation = "https://docs.rs/azure_data_cosmos" keywords = ["sdk", "azure", "rest", "iot", "cloud"] categories = ["api-bindings"] edition = "2021" -rust-version = "1.64.0" [dependencies] async-trait = "0.1" diff --git a/sdk/data_tables/Cargo.toml b/sdk/data_tables/Cargo.toml index 9f72b8496a..5b43530a0f 100644 --- a/sdk/data_tables/Cargo.toml +++ b/sdk/data_tables/Cargo.toml @@ -11,7 +11,6 @@ documentation = "https://docs.rs/azure_data_tables" keywords = ["sdk", "azure", "storage", "data-tables"] categories = ["api-bindings"] edition = "2021" -rust-version = "1.64.0" [dependencies] azure_core = { path = "../core", version = "0.13" } diff --git a/sdk/iot_hub/Cargo.toml b/sdk/iot_hub/Cargo.toml index 2262d2ce11..ba3b37c35a 100644 --- a/sdk/iot_hub/Cargo.toml +++ b/sdk/iot_hub/Cargo.toml @@ -8,7 +8,6 @@ license = "MIT" repository = "https://github.com/azure/azure-sdk-for-rust" homepage = "https://github.com/azure/azure-sdk-for-rust" documentation = "https://docs.rs/azure_iot_hub" -rust-version = "1.64.0" [dependencies] async-trait = "0.1" diff --git a/sdk/security_keyvault/Cargo.toml b/sdk/security_keyvault/Cargo.toml index 3dddf33572..c2f1bbc2f5 100644 --- a/sdk/security_keyvault/Cargo.toml +++ b/sdk/security_keyvault/Cargo.toml @@ -11,7 +11,6 @@ categories = ["api-bindings"] readme = "README.md" license = "MIT" edition = "2021" -rust-version = "1.64.0" [dependencies] async-trait = "0.1" diff --git a/sdk/storage/Cargo.toml b/sdk/storage/Cargo.toml index bbc793640b..57c73325cf 100644 --- a/sdk/storage/Cargo.toml +++ b/sdk/storage/Cargo.toml @@ -11,7 +11,6 @@ documentation = "https://docs.rs/azure_storage" keywords = ["sdk", "azure", "storage"] categories = ["api-bindings"] edition = "2021" -rust-version = "1.64.0" [dependencies] async-trait = "0.1" diff --git a/sdk/storage_blobs/Cargo.toml b/sdk/storage_blobs/Cargo.toml index 1c960de14f..b40c34cc4f 100644 --- a/sdk/storage_blobs/Cargo.toml +++ b/sdk/storage_blobs/Cargo.toml @@ -11,7 +11,6 @@ documentation = "https://docs.rs/azure_storage_blobs" keywords = ["sdk", "azure", "storage", "blobs"] categories = ["api-bindings"] edition = "2021" -rust-version = "1.64.0" [dependencies] azure_core = { path = "../core", version = "0.13", features = ["xml"] } diff --git a/sdk/storage_datalake/Cargo.toml b/sdk/storage_datalake/Cargo.toml index 3430ae7919..0a4365e22d 100644 --- a/sdk/storage_datalake/Cargo.toml +++ b/sdk/storage_datalake/Cargo.toml @@ -11,7 +11,6 @@ documentation = "https://docs.rs/azure_storage_datalake" keywords = ["sdk", "azure", "storage", "datalake"] categories = ["api-bindings"] edition = "2021" -rust-version = "1.64.0" [dependencies] async-trait = "0.1" diff --git a/sdk/storage_queues/Cargo.toml b/sdk/storage_queues/Cargo.toml index f7b097cf86..c8cc136048 100644 --- a/sdk/storage_queues/Cargo.toml +++ b/sdk/storage_queues/Cargo.toml @@ -11,7 +11,6 @@ documentation = "https://docs.rs/azure_storage_queues" keywords = ["sdk", "azure", "storage", "queues"] categories = ["api-bindings"] edition = "2021" -rust-version = "1.64.0" [dependencies] azure_core = { path = "../core", version = "0.13", default-features=false, features = ["xml"] } From ec3c321f49a045a36a1eb2e6d7501c8c3b3aa874 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 10:54:38 -0400 Subject: [PATCH 02/11] try removing msrv from matrix strategy --- .github/workflows/build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1dbdeb31e9..18e90e3bc9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,7 +30,6 @@ jobs: # we don't care about code style from nightly build: - stable - - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 @@ -46,7 +45,6 @@ jobs: build: - stable - nightly - - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 @@ -62,7 +60,6 @@ jobs: build: - stable - nightly - - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 @@ -81,7 +78,6 @@ jobs: build: - stable - nightly - - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 @@ -100,7 +96,6 @@ jobs: build: - stable - nightly - - ${{ needs.msrv.outputs.msrv }}} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 From e1ff1367a6eabc8cd4ab1eb9cdfcdb4c0a6eff85 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 10:55:26 -0400 Subject: [PATCH 03/11] fix syntax error --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18e90e3bc9..62d32ed97c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,8 +17,8 @@ jobs: steps: - uses: actions/checkout@v3 - run: | - MSRV=$(cargo metadata --format-version=1 --no-deps | jq '.packages[] | select(.name == "azure_core").rust_version' -r) - echo ::set-output name=msrv::${MSRV} + MSRV=$(cargo metadata --format-version=1 --no-deps | jq '.packages[] | select(.name == "azure_core").rust_version' -r) + echo ::set-output name=msrv::${MSRV} code-style: name: Code Style From 46409d03b4e7def49737f6ffbfe03a00185a92c4 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 11:02:48 -0400 Subject: [PATCH 04/11] fix set-output syntax --- .github/workflows/build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62d32ed97c..3007acf07d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,12 +13,14 @@ jobs: name: Get minimum supported rust version runs-on: ubuntu-20.04 outputs: - msrv: ${{ steps.msrv.outputs.msrv }} + msrv: ${{ steps.get_msrv.outputs.msrv }} steps: - uses: actions/checkout@v3 - - run: | + - id: get_msrv + run: | MSRV=$(cargo metadata --format-version=1 --no-deps | jq '.packages[] | select(.name == "azure_core").rust_version' -r) - echo ::set-output name=msrv::${MSRV} + echo setting msrv to ${MSRV} + echo msrv=${MSRV} >> "$GITHUB_OUTPUT" code-style: name: Code Style From 9293807434db211c996d9ccc07114bd9ad3db50f Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 11:03:45 -0400 Subject: [PATCH 05/11] remove extra } --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3007acf07d..9cbb43c414 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,7 +101,7 @@ jobs: steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 - - run: eng/scripts/e2e_tests.sh ${{ matrix.build }}} + - run: eng/scripts/e2e_tests.sh ${{ matrix.build }} - name: display free disk space run: df -h / if: ${{ always() }} From 878dbfc8f17e7eaf3cbc77f5912725e49cba3185 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 11:06:18 -0400 Subject: [PATCH 06/11] add using msrv for codestyle --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9cbb43c414..f7c0d00f20 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,6 +32,7 @@ jobs: # we don't care about code style from nightly build: - stable + - ${{ needs.msrv.outputs.msrv }} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 From fba51b193579cec9acc6a4ad17bd0e5612e07b41 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 11:16:27 -0400 Subject: [PATCH 07/11] matrix off of `msrv` task across the pipeline --- .github/workflows/build.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f7c0d00f20..7296e9a480 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,18 +25,11 @@ jobs: code-style: name: Code Style runs-on: ubuntu-20.04 - needs: - - msrv - strategy: - matrix: - # we don't care about code style from nightly - build: - - stable - - ${{ needs.msrv.outputs.msrv }} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 - - run: eng/scripts/code_style.sh ${{ matrix.build }} + # for code style, we only care about `stable` + - run: eng/scripts/code_style.sh stable test-sdk: name: SDK Tests @@ -48,6 +41,7 @@ jobs: build: - stable - nightly + - ${{ needs.msrv.outputs.msrv }} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 @@ -63,6 +57,7 @@ jobs: build: - stable - nightly + - ${{ needs.msrv.outputs.msrv }} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 @@ -81,6 +76,7 @@ jobs: build: - stable - nightly + - ${{ needs.msrv.outputs.msrv }} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 @@ -99,6 +95,7 @@ jobs: build: - stable - nightly + - ${{ needs.msrv.outputs.msrv }} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 @@ -125,4 +122,5 @@ jobs: with: repository: OAI/OpenAPI-Specification path: OpenAPI-Specification + # for code style, we only care about `stable` - run: azure-sdk-for-rust/eng/scripts/autorust_tests.sh From 85a316918e6d62fd38b532ffcb371ca6cfaac164 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 11:21:34 -0400 Subject: [PATCH 08/11] install required components --- eng/scripts/code_style.sh | 1 + eng/scripts/services_tests.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/eng/scripts/code_style.sh b/eng/scripts/code_style.sh index efa6bf8c0c..a5f827c915 100755 --- a/eng/scripts/code_style.sh +++ b/eng/scripts/code_style.sh @@ -9,6 +9,7 @@ export RUSTDOCFLAGS="-D warnings" export RUSTFLAGS="-Dwarnings" rustup update --no-self-update ${BUILD} +rustup component add rustfmt --toolchain ${BUILD} cargo +${BUILD} install cargo-readme cargo +${BUILD} fmt --all -- --check cargo +${BUILD} clippy --all diff --git a/eng/scripts/services_tests.sh b/eng/scripts/services_tests.sh index 8c0bb89f30..b6b9cbf3c6 100755 --- a/eng/scripts/services_tests.sh +++ b/eng/scripts/services_tests.sh @@ -6,6 +6,7 @@ cd $(dirname ${BASH_SOURCE[0]})/../../ BUILD=${1:-stable} rustup update --no-self-update ${BUILD} +rustup component add clippy --toolchain ${BUILD} export RUSTFLAGS="-Dwarnings -Aunreachable-code -Aunused-assignments -Adead-code -Aclippy::new-without-default -Aclippy::unnecessary_to_owned" cargo +${BUILD} check --manifest-path services/Cargo.toml --all cargo +${BUILD} check --manifest-path services/Cargo.toml --examples From 14b80c1ae003726c971e834c6a287b1ab8a8b0d1 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 11:22:28 -0400 Subject: [PATCH 09/11] bump msrv to 1.65.0 due to time-core dependency --- sdk/core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/Cargo.toml b/sdk/core/Cargo.toml index efc0467e03..a13bb8e24d 100644 --- a/sdk/core/Cargo.toml +++ b/sdk/core/Cargo.toml @@ -11,7 +11,7 @@ documentation = "https://docs.rs/azure_core" keywords = ["sdk", "azure", "rest", "iot", "cloud"] categories = ["api-bindings"] edition = "2021" -rust-version = "1.64.0" +rust-version = "1.65.0" [dependencies] async-trait = "0.1" From 8e1d971f0c9439cf47344a295880335b6415471a Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 11:27:19 -0400 Subject: [PATCH 10/11] update MSRV to 1.67.0 due to dependency on `time` --- sdk/core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/Cargo.toml b/sdk/core/Cargo.toml index a13bb8e24d..ba200b0523 100644 --- a/sdk/core/Cargo.toml +++ b/sdk/core/Cargo.toml @@ -11,7 +11,7 @@ documentation = "https://docs.rs/azure_core" keywords = ["sdk", "azure", "rest", "iot", "cloud"] categories = ["api-bindings"] edition = "2021" -rust-version = "1.65.0" +rust-version = "1.67.0" [dependencies] async-trait = "0.1" From 4e2e078ab4af6316ce6c0a7b9df45840a0404a3d Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 16 Aug 2023 11:59:43 -0400 Subject: [PATCH 11/11] ensure rustfmt is installed for testing services --- eng/scripts/services_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/services_tests.sh b/eng/scripts/services_tests.sh index b6b9cbf3c6..6b454aaa1e 100755 --- a/eng/scripts/services_tests.sh +++ b/eng/scripts/services_tests.sh @@ -6,7 +6,7 @@ cd $(dirname ${BASH_SOURCE[0]})/../../ BUILD=${1:-stable} rustup update --no-self-update ${BUILD} -rustup component add clippy --toolchain ${BUILD} +rustup component add rustfmt clippy --toolchain ${BUILD} export RUSTFLAGS="-Dwarnings -Aunreachable-code -Aunused-assignments -Adead-code -Aclippy::new-without-default -Aclippy::unnecessary_to_owned" cargo +${BUILD} check --manifest-path services/Cargo.toml --all cargo +${BUILD} check --manifest-path services/Cargo.toml --examples