From 10d0bacdeb9d61549f72ac6281cb98a774c77cab Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 7 Mar 2023 11:27:11 +0000 Subject: [PATCH 1/7] Update to latest golden images. Includes https://github.com/saltstack/salt-jenkins/pull/1802 Signed-off-by: Pedro Algarvio --- cicd/amis.yml | 30 +++++++------- cicd/golden-images.json | 92 ++++++++++++++++++++--------------------- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/cicd/amis.yml b/cicd/amis.yml index 7d7cf46a3f18..f1a3b0f5a9fc 100644 --- a/cicd/amis.yml +++ b/cicd/amis.yml @@ -1,15 +1,15 @@ -alma-8-x86_64: ami-0605535c900176e44 -amazon-2-x86_64: ami-0b85ffe08394a6e7c -arch-lts-x86_64: ami-063db28c7923c95f9 -centos-7-x86_64: ami-0c114f33cd54afb03 -centosstream-9-x86_64: ami-0eeefe9055b5d9630 -debian-10-amd64: ami-01664f466e18c0803 -debian-11-arm64: ami-07b235058177e8c22 -debian-11-amd64: ami-032287fde000ed111 -opensuse-15-x86_64: ami-06014e2cb5d08dba2 -photon-3-x86_64: ami-0868c9e1ac7bab552 -ubuntu-2004-arm64: ami-050ef2a49e7ab25d9 -ubuntu-2004-amd64: ami-0dd91400b62731387 -ubuntu-2204-amd64: ami-054df5d9f56f2144b -windows-2016-x64: ami-005f1bcbecd69fec9 -windows-2019-x64: ami-0f972bd95e3d7671d +alma-8-x86_64: ami-0e38e779c7e9a58cb +amazon-2-x86_64: ami-0808ff1e4bd65c26e +arch-lts-x86_64: ami-05cb9be3cdb8f9ded +centos-7-x86_64: ami-062de6e6b1bea1987 +centosstream-9-x86_64: ami-0a8429c3f5f933d85 +debian-10-amd64: ami-01c864e5f37943e82 +debian-11-arm64: ami-0f0687d3bed6d0f9e +debian-11-amd64: ami-097a7bcb88c641a4f +opensuse-15-x86_64: ami-053085867e8e2a437 +photon-3-x86_64: ami-0c60abf0038288d5b +ubuntu-2004-arm64: ami-090c51d35099a31fc +ubuntu-2004-amd64: ami-0d6b6f0ee825d6cc0 +ubuntu-2204-amd64: ami-0eba94938aad408da +windows-2016-x64: ami-09e918f01701cbaca +windows-2019-x64: ami-072893e306bcedd6e diff --git a/cicd/golden-images.json b/cicd/golden-images.json index 8789e104392e..ed3b9f8deac4 100644 --- a/cicd/golden-images.json +++ b/cicd/golden-images.json @@ -1,8 +1,8 @@ { "almalinux-8": { - "ami": "ami-0605535c900176e44", + "ami": "ami-0e38e779c7e9a58cb", "ami_description": "CI Image of AlmaLinux 8 x86_64", - "ami_name": "salt-project/ci/almalinux/8/x86_64/20230220.1156", + "ami_name": "salt-project/ci/almalinux/8/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -10,9 +10,9 @@ "ssh_username": "ec2-user" }, "almalinux-9": { - "ami": "ami-0bfbbfb035a16aa67", + "ami": "ami-0e478b3a983a8aa3a", "ami_description": "CI Image of AlmaLinux 9 x86_64", - "ami_name": "salt-project/ci/almalinux/9/x86_64/20230220.1156", + "ami_name": "salt-project/ci/almalinux/9/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -20,9 +20,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2": { - "ami": "ami-0b85ffe08394a6e7c", + "ami": "ami-0808ff1e4bd65c26e", "ami_description": "CI Image of AmazonLinux 2 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20230220.1156", + "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -30,9 +30,9 @@ "ssh_username": "ec2-user" }, "archlinux-lts": { - "ami": "ami-063db28c7923c95f9", + "ami": "ami-05cb9be3cdb8f9ded", "ami_description": "CI Image of ArchLinux lts x86_64", - "ami_name": "salt-project/ci/archlinux/lts/x86_64/20230220.1156", + "ami_name": "salt-project/ci/archlinux/lts/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "false", "instance_type": "t3a.large", @@ -40,9 +40,9 @@ "ssh_username": "arch" }, "centos-7": { - "ami": "ami-0c114f33cd54afb03", + "ami": "ami-062de6e6b1bea1987", "ami_description": "CI Image of CentOS 7 x86_64", - "ami_name": "salt-project/ci/centos/7/x86_64/20230220.1156", + "ami_name": "salt-project/ci/centos/7/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -50,9 +50,9 @@ "ssh_username": "centos" }, "centosstream-8": { - "ami": "ami-0a75815b2af074b46", + "ami": "ami-00b1e2790dced55ba", "ami_description": "CI Image of CentOSStream 8 x86_64", - "ami_name": "salt-project/ci/centosstream/8/x86_64/20230220.1156", + "ami_name": "salt-project/ci/centosstream/8/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -60,9 +60,9 @@ "ssh_username": "cloud-user" }, "centosstream-9": { - "ami": "ami-0eeefe9055b5d9630", + "ami": "ami-0a8429c3f5f933d85", "ami_description": "CI Image of CentOSStream 9 x86_64", - "ami_name": "salt-project/ci/centosstream/9/x86_64/20230220.1156", + "ami_name": "salt-project/ci/centosstream/9/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -70,9 +70,9 @@ "ssh_username": "ec2-user" }, "debian-10": { - "ami": "ami-01664f466e18c0803", + "ami": "ami-01c864e5f37943e82", "ami_description": "CI Image of Debian 10 x86_64", - "ami_name": "salt-project/ci/debian/10/x86_64/20230220.1156", + "ami_name": "salt-project/ci/debian/10/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -80,9 +80,9 @@ "ssh_username": "admin" }, "debian-11-arm64": { - "ami": "ami-07b235058177e8c22", + "ami": "ami-0f0687d3bed6d0f9e", "ami_description": "CI Image of Debian 11 arm64", - "ami_name": "salt-project/ci/debian/11/arm64/20230220.1156", + "ami_name": "salt-project/ci/debian/11/arm64/20230306.1703", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -90,9 +90,9 @@ "ssh_username": "admin" }, "debian-11": { - "ami": "ami-032287fde000ed111", + "ami": "ami-097a7bcb88c641a4f", "ami_description": "CI Image of Debian 11 x86_64", - "ami_name": "salt-project/ci/debian/11/x86_64/20230220.1156", + "ami_name": "salt-project/ci/debian/11/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -100,9 +100,9 @@ "ssh_username": "admin" }, "fedora-36": { - "ami": "ami-0139632eacf432cb8", + "ami": "ami-04d3d4001c7eb6084", "ami_description": "CI Image of Fedora 36 x86_64", - "ami_name": "salt-project/ci/fedora/36/x86_64/20230220.1156", + "ami_name": "salt-project/ci/fedora/36/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -110,9 +110,9 @@ "ssh_username": "fedora" }, "fedora-37": { - "ami": "ami-0bd4d6d08c9e59606", + "ami": "ami-047f01d4d1b8eade2", "ami_description": "CI Image of Fedora 37 x86_64", - "ami_name": "salt-project/ci/fedora/37/x86_64/20230220.1156", + "ami_name": "salt-project/ci/fedora/37/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -120,9 +120,9 @@ "ssh_username": "fedora" }, "fedora-38": { - "ami": "ami-04faf7622a63a6632", + "ami": "ami-04600a36ca6b94c1c", "ami_description": "CI Image of Fedora 38 x86_64", - "ami_name": "salt-project/ci/fedora/38/x86_64/20230220.1156", + "ami_name": "salt-project/ci/fedora/38/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -130,9 +130,9 @@ "ssh_username": "fedora" }, "opensuse-15": { - "ami": "ami-06014e2cb5d08dba2", + "ami": "ami-053085867e8e2a437", "ami_description": "CI Image of Opensuse 15 x86_64", - "ami_name": "salt-project/ci/opensuse/15/x86_64/20230220.1156", + "ami_name": "salt-project/ci/opensuse/15/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -140,9 +140,9 @@ "ssh_username": "ec2-user" }, "photonos-3": { - "ami": "ami-0868c9e1ac7bab552", + "ami": "ami-0c60abf0038288d5b", "ami_description": "CI Image of PhotonOS 3 x86_64", - "ami_name": "salt-project/ci/photonos/3/x86_64/20230220.1156", + "ami_name": "salt-project/ci/photonos/3/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -150,9 +150,9 @@ "ssh_username": "root" }, "photonos-4": { - "ami": "ami-0bd5723a927300435", + "ami": "ami-0f05edd4afe1b852b", "ami_description": "CI Image of PhotonOS 4 x86_64", - "ami_name": "salt-project/ci/photonos/4/x86_64/20230220.1156", + "ami_name": "salt-project/ci/photonos/4/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -160,9 +160,9 @@ "ssh_username": "root" }, "ubuntu-20.04-arm64": { - "ami": "ami-050ef2a49e7ab25d9", + "ami": "ami-090c51d35099a31fc", "ami_description": "CI Image of Ubuntu 20.04 arm64", - "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20230220.1156", + "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20230306.1703", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -170,9 +170,9 @@ "ssh_username": "ubuntu" }, "ubuntu-20.04": { - "ami": "ami-0dd91400b62731387", + "ami": "ami-0d6b6f0ee825d6cc0", "ami_description": "CI Image of Ubuntu 20.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20230220.1156", + "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -180,9 +180,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04-arm64": { - "ami": "ami-0e185c664438a374f", + "ami": "ami-01b52de1f19e79343", "ami_description": "CI Image of Ubuntu 22.04 arm64", - "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20230220.1156", + "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20230306.1703", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -190,9 +190,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04": { - "ami": "ami-054df5d9f56f2144b", + "ami": "ami-0eba94938aad408da", "ami_description": "CI Image of Ubuntu 22.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20230220.1156", + "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20230306.1704", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -200,9 +200,9 @@ "ssh_username": "ubuntu" }, "windows-2016": { - "ami": "ami-005f1bcbecd69fec9", + "ami": "ami-09e918f01701cbaca", "ami_description": "CI Image of Windows 2016 x86_64", - "ami_name": "salt-project/ci/windows/2016/x86_64/20230220.1156", + "ami_name": "salt-project/ci/windows/2016/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -210,9 +210,9 @@ "ssh_username": "Administrator" }, "windows-2019": { - "ami": "ami-0f972bd95e3d7671d", + "ami": "ami-072893e306bcedd6e", "ami_description": "CI Image of Windows 2019 x86_64", - "ami_name": "salt-project/ci/windows/2019/x86_64/20230220.1311", + "ami_name": "salt-project/ci/windows/2019/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -220,9 +220,9 @@ "ssh_username": "Administrator" }, "windows-2022": { - "ami": "ami-0c99d04af93b78866", + "ami": "ami-0cfbb2e681636b902", "ami_description": "CI Image of Windows 2022 x86_64", - "ami_name": "salt-project/ci/windows/2022/x86_64/20230220.1156", + "ami_name": "salt-project/ci/windows/2022/x86_64/20230306.1703", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", From 4e707cba04e4b9e3ce4110c4af6dc748322a7f6b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 7 Mar 2023 21:15:29 +0000 Subject: [PATCH 2/7] Allow jobs on forks which have self hosted runners setup. Signed-off-by: Pedro Algarvio --- tools/ci.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/ci.py b/tools/ci.py index 5bf725f83f62..e074ff9b2c57 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -196,7 +196,10 @@ def runner_types(ctx: Context, event_name: str): # This is a push or a scheduled event ctx.info(f"Running from a {event_name!r} event") - if gh_event["repository"]["fork"] is True: + if ( + gh_event["repository"]["fork"] is True + and os.environ.get("FORK_HAS_SELF_HOSTED_RUNNERS", "0") == "1" + ): # This is running on a forked repository, don't run tests ctx.info("The push event is on a forked repository") runners["github-hosted"] = True @@ -205,7 +208,7 @@ def runner_types(ctx: Context, event_name: str): wfh.write(f"runners={json.dumps(runners)}\n") ctx.exit(0) - # Not running on a fork, run everything + # Not running on a fork, or the fork has self hosted runners, run everything ctx.info(f"The {event_name!r} event is from the main repository") runners["github-hosted"] = runners["self-hosted"] = True ctx.info("Writing 'runners' to the github outputs file") From af5e2e339040198550f22887f601317113546015 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 8 Mar 2023 07:34:09 +0000 Subject: [PATCH 3/7] Try to get the VM environment from the instance tags Signed-off-by: Pedro Algarvio --- .github/workflows/test-action.yml | 17 ++++++++++-- .github/workflows/test-packages-action.yml | 17 ++++++++++-- .pre-commit-config.yaml | 1 + tools/vm.py | 32 ++++++++++++++++++++-- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index e30395b96b04..6c978f38c7a3 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -137,11 +137,18 @@ jobs: echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> "$GITHUB_ENV" fi + - name: Get Salt Project GitHub Actions Bot Environment + if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" + - name: Start VM if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' id: spin-up-vm run: | - tools --timestamps vm create --retries=2 ${{ inputs.distro-slug }} + tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }} - name: List Free Space if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' @@ -241,12 +248,18 @@ jobs: with: name: testrun-changed-files.txt + - name: Get Salt Project GitHub Actions Bot Environment + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" + - name: Start VM id: spin-up-vm env: TESTS_CHUNK: ${{ matrix.tests-chunk }} run: | - tools --timestamps vm create --retries=2 ${{ inputs.distro-slug }} + tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }} - name: List Free Space run: | diff --git a/.github/workflows/test-packages-action.yml b/.github/workflows/test-packages-action.yml index 1496e02330e3..ae4e53df3c1a 100644 --- a/.github/workflows/test-packages-action.yml +++ b/.github/workflows/test-packages-action.yml @@ -110,11 +110,18 @@ jobs: if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' uses: ./.github/actions/setup-python-tools-scripts + - name: Get Salt Project GitHub Actions Bot Environment + if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" + - name: Start VM if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' id: spin-up-vm run: | - tools --timestamps vm create --retries=2 ${{ inputs.distro-slug }} + tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }} - name: List Free Space if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' @@ -216,10 +223,16 @@ jobs: - name: Setup Python Tools Scripts uses: ./.github/actions/setup-python-tools-scripts + - name: Get Salt Project GitHub Actions Bot Environment + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" + - name: Start VM id: spin-up-vm run: | - tools --timestamps vm create --retries=2 ${{ inputs.distro-slug }} + tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }} - name: List Free Space run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1124ffeb7ba5..58dbbe762572 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1330,6 +1330,7 @@ repos: - rich - types-attrs - types-pyyaml + - types-requests - repo: https://github.com/saltstack/mirrors-nox rev: v2021.6.12 diff --git a/tools/vm.py b/tools/vm.py index 10244843e5b9..bb0d65d47b77 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, cast from ptscripts import Context, command_group +from requests.exceptions import ConnectTimeout import tools.utils @@ -95,8 +96,13 @@ "help": "How many times to retry creating and connecting to a vm", }, "environment": { - "help": "The AWS environment to use.", - "choices": ("prod", "test"), + "help": ( + "The AWS environment to use. When the value is auto, an " + "attempt will be made to get the right environment from the " + "AWS instance metadata endpoint. This only works for bastion " + "VMs." + ), + "choices": ("prod", "test", "auto"), }, } ) @@ -117,6 +123,28 @@ def create( ctx.exit(1, "We need a key name to spin a VM") if not retries: retries = 1 + if environment == "auto": + # Lets get the environment from the instance profile if we're on a bastion VM + with ctx.web as web: + try: + ret = web.put( + "http://169.254.169.254/latest/api/token", + headers={"X-aws-ec2-metadata-token-ttl-seconds": "10"}, + timeout=1, + ) + token = ret.text.strip() + ret = web.get( + "http://169.254.169.254/latest/meta-data/tags/instance/spb:environment", + headers={"X-aws-ec2-metadata-token": token}, + ) + spb_environment = ret.text.strip() + if spb_environment: + ctx.info(f"Discovered VM environment: {spb_environment}") + environment = spb_environment + except ConnectTimeout: + # We're apparently not in bastion VM + environment = None + attempts = 0 while True: attempts += 1 From ebe6c4f6d51702a0a23cff636f6621e5efeb3e66 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 8 Mar 2023 15:23:57 +0000 Subject: [PATCH 4/7] Switch to using our onedir to run the package tests Signed-off-by: Pedro Algarvio --- .github/workflows/test-action-macos.yml | 1 + .github/workflows/test-action.yml | 1 - .../workflows/test-packages-action-macos.yml | 14 ++++- .github/workflows/test-packages-action.yml | 1 - noxfile.py | 57 ++++++++++++++++++- tools/ci.py | 6 +- 6 files changed, 71 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 3333930dae38..05d6560080fa 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -114,6 +114,7 @@ jobs: python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" cd artifacts tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz + - name: Set up Python ${{ inputs.python-version }} if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' uses: actions/setup-python@v4 diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 6c978f38c7a3..72f29429c41a 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -103,7 +103,6 @@ jobs: path: nox.${{ inputs.distro-slug }}.tar.* key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} - # Skip jobs if nox.*.tar.* is already cached - name: Download Onedir Tarball as an Artifact if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' uses: actions/download-artifact@v3 diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index d59072436f0c..82b68eb8189b 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -89,7 +89,6 @@ jobs: path: nox.${{ inputs.distro-slug }}.tar.* key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ matrix.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} - # Skip jobs if nox.*.tar.* is already cached - name: Download Onedir Tarball as an Artifact if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' uses: actions/download-artifact@v3 @@ -187,6 +186,19 @@ jobs: run: | tree pkg/artifacts + - name: Download Onedir Tarball as an Artifact + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz + path: artifacts/ + + - name: Decompress Onedir Tarball + shell: bash + run: | + python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" + cd artifacts + tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz + - name: Set up Python ${{ inputs.python-version }} uses: actions/setup-python@v4 with: diff --git a/.github/workflows/test-packages-action.yml b/.github/workflows/test-packages-action.yml index ae4e53df3c1a..4a0eeb718f39 100644 --- a/.github/workflows/test-packages-action.yml +++ b/.github/workflows/test-packages-action.yml @@ -90,7 +90,6 @@ jobs: path: nox.${{ inputs.distro-slug }}.tar.* key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ matrix.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} - # Skip jobs if nox.*.tar.* is already cached - name: Download Onedir Tarball as an Artifact if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' uses: actions/download-artifact@v3 diff --git a/noxfile.py b/noxfile.py index 18934c08fc34..21d017666d17 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1762,12 +1762,14 @@ def build(session): session.run("python", "-m", "twine", "check", "dist/*") -def _pkg_test(session, cmd_args, test_type): +def _pkg_test(session, cmd_args, test_type, onedir=False): pydir = _get_pydir(session) junit_report_filename = f"test-results-{test_type}" runtests_log_filename = f"runtests-{test_type}" # Install requirements - if _upgrade_pip_setuptools_and_wheel(session): + if onedir and IS_LINUX: + session_run_always(session, "python3", "-m", "relenv", "toolchain", "fetch") + if _upgrade_pip_setuptools_and_wheel(session, onedir=onedir): if IS_WINDOWS: file_name = "pkgtests-windows.txt" else: @@ -1780,6 +1782,10 @@ def _pkg_test(session, cmd_args, test_type): install_command = ["--progress-bar=off", "-r", requirements_file] session.install(*install_command, silent=PIP_INSTALL_SILENT) + env = {} + if onedir: + env["ONEDIR_TESTRUN"] = "1" + pytest_args = ( cmd_args[:] + [ @@ -1788,7 +1794,7 @@ def _pkg_test(session, cmd_args, test_type): ] + session.posargs ) - _pytest(session, False, pytest_args) + _pytest(session, False, pytest_args, env=env) @nox.session(python=_PYTHON_VERSIONS, name="test-pkgs") @@ -1799,6 +1805,21 @@ def test_pkgs(session): _pkg_test(session, ["pkg/tests/"], "pkg") +@nox.session( + python=str(ONEDIR_PYTHON_PATH), + name="test-pkgs-onedir", + venv_params=["--system-site-packages"], +) +def test_pkgs_onedir(session): + if not ONEDIR_ARTIFACT_PATH.exists(): + session.error( + "The salt onedir artifact, expected to be in '{}', was not found".format( + ONEDIR_ARTIFACT_PATH.relative_to(REPO_ROOT) + ) + ) + _pkg_test(session, ["pkg/tests/"], "pkg", onedir=True) + + @nox.session(python=_PYTHON_VERSIONS, name="test-upgrade-pkgs") @nox.parametrize("classic", [False, True]) def test_upgrade_pkgs(session, classic): @@ -1823,3 +1844,33 @@ def test_upgrade_pkgs(session, classic): cmd_args = ["pkg/tests/", "--no-install"] + session.posargs _pkg_test(session, cmd_args, test_type) + + +@nox.session( + python=str(ONEDIR_PYTHON_PATH), + name="test-upgrade-pkgs-onedir", + venv_params=["--system-site-packages"], +) +@nox.parametrize("classic", [False, True]) +def test_upgrade_pkgs_onedir(session, classic): + """ + pytest pkg upgrade tests session + """ + test_type = "pkg_upgrade" + cmd_args = [ + "pkg/tests/upgrade/test_salt_upgrade.py::test_salt_upgrade", + "--upgrade", + "--no-uninstall", + ] + if classic: + cmd_args = cmd_args + ["--classic"] + # Workaround for installing and running classic packages from 3005.1 + # They can only run with importlib-metadata<5.0.0. + subprocess.run(["pip3", "install", "importlib-metadata==4.13.0"], check=False) + try: + _pkg_test(session, cmd_args, test_type, onedir=True) + except nox.command.CommandFailed: + sys.exit(1) + + cmd_args = ["pkg/tests/", "--no-install"] + session.posargs + _pkg_test(session, cmd_args, test_type, onedir=True) diff --git a/tools/ci.py b/tools/ci.py index e074ff9b2c57..645051dc6039 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -670,7 +670,7 @@ def pkg_matrix(ctx: Context, distro_slug: str, pkg_type: str): """ _matrix = [] sessions = [ - "test-pkgs-3", + "test-pkgs-onedir", ] if ( distro_slug @@ -686,13 +686,13 @@ def pkg_matrix(ctx: Context, distro_slug: str, pkg_type: str): # we will need to ensure when we release 3006.0 # we allow for 3006.0 jobs to run, because then # we will have arm64 onedir packages to upgrade from - sessions.append("'test-upgrade-pkgs-3(classic=False)'") + sessions.append("'test-upgrade-pkgs-onedir(classic=False)'") if ( distro_slug not in ["centosstream-9", "ubuntu-22.04", "ubuntu-22.04-arm64"] and "MSI" != pkg_type ): # Packages for these OSs where never built for classic previously - sessions.append("'test-upgrade-pkgs-3(classic=True)'") + sessions.append("'test-upgrade-pkgs-onedir(classic=True)'") for sess in sessions: _matrix.append({"nox-session": sess}) From b7d8a30ccc2a9a4e1d17421e47315a2b5bc263e1 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 8 Mar 2023 17:06:42 +0000 Subject: [PATCH 5/7] Remove nox sessions and requirements files which are no longer necessary Signed-off-by: Pedro Algarvio --- .pre-commit-config.yaml | 79 +------- noxfile.py | 34 ---- .../static/ci/py3.7/pkgtests-windows.txt | 184 ------------------ requirements/static/ci/py3.7/pkgtests.txt | 166 ---------------- .../static/ci/py3.8/pkgtests-windows.txt | 173 ---------------- requirements/static/ci/py3.8/pkgtests.txt | 155 --------------- .../static/ci/py3.9/pkgtests-windows.txt | 169 ---------------- requirements/static/ci/py3.9/pkgtests.txt | 151 -------------- 8 files changed, 1 insertion(+), 1110 deletions(-) delete mode 100644 requirements/static/ci/py3.7/pkgtests-windows.txt delete mode 100644 requirements/static/ci/py3.7/pkgtests.txt delete mode 100644 requirements/static/ci/py3.8/pkgtests-windows.txt delete mode 100644 requirements/static/ci/py3.8/pkgtests.txt delete mode 100644 requirements/static/ci/py3.9/pkgtests-windows.txt delete mode 100644 requirements/static/ci/py3.9/pkgtests.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 58dbbe762572..719edae7b7b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -968,45 +968,6 @@ repos: # <---- Invoke ----------------------------------------------------------------------------------------------------- # <---- PKG ci requirements----------------------------------------------------------------------------------------- - - id: pip-tools-compile - alias: compile-ci-pkg-3.7-requirements - name: PKG tests CI Py3.7 Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/(pkg/linux\.in|ci/((pkgtests|common)\.in|py3\.7/pkgtests\.in)))$ - pass_filenames: false - args: - - -v - - --py-version=3.7 - - --platform=linux - - --include=requirements/base.txt - - --include=requirements/zeromq.txt - - requirements/static/ci/pkgtests.in - - - id: pip-tools-compile - alias: compile-ci-pkg-3.8-requirements - name: PKG tests CI Py3.8 Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/(pkg/linux\.in|ci/((pkgtests|common)\.in|py3\.8/pkgtests\.in)))$ - pass_filenames: false - args: - - -v - - --py-version=3.8 - - --platform=linux - - --include=requirements/base.txt - - --include=requirements/zeromq.txt - - requirements/static/ci/pkgtests.in - - - id: pip-tools-compile - alias: compile-ci-pkg-3.9-requirements - name: PKG tests CI Py3.9 Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/(pkg/linux\.in|ci/((pkgtests|common)\.in|py3\.9/pkgtests\.in)))$ - pass_filenames: false - args: - - -v - - --py-version=3.9 - - --platform=linux - - --include=requirements/base.txt - - --include=requirements/zeromq.txt - - requirements/static/ci/pkgtests.in - - id: pip-tools-compile alias: compile-ci-pkg-3.10-requirements name: PKG tests CI Py3.10 Requirements @@ -1020,45 +981,6 @@ repos: - --include=requirements/zeromq.txt - requirements/static/ci/pkgtests.in - - id: pip-tools-compile - alias: compile-ci-windows-pkg-3.7-requirements - name: PKG tests Windows CI Py3.7 Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/(pkg/linux\.in|ci/((pkgtests-windows|common)\.in|py3\.7/pkgtests-windows\.in)))$ - pass_filenames: false - args: - - -v - - --py-version=3.7 - - --platform=windows - - --include=requirements/base.txt - - --include=requirements/zeromq.txt - - requirements/static/ci/pkgtests-windows.in - - - id: pip-tools-compile - alias: compile-ci-windows-pkg-3.8-requirements - name: PKG tests Windows CI Py3.8 Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/(pkg/linux\.in|ci/((pkgtests-windows|common)\.in|py3\.8/pkgtests-windows\.in)))$ - pass_filenames: false - args: - - -v - - --py-version=3.8 - - --platform=windows - - --include=requirements/base.txt - - --include=requirements/zeromq.txt - - requirements/static/ci/pkgtests-windows.in - - - id: pip-tools-compile - alias: compile-ci-windows-pkg-3.9-requirements - name: PKG tests Windows CI Py3.9 Requirements - files: ^requirements/((base|zeromq|pytest)\.txt|static/(pkg/linux\.in|ci/((pkgtests-windows|common)\.in|py3\.9/pkgtests-windows\.in)))$ - pass_filenames: false - args: - - -v - - --py-version=3.9 - - --platform=windows - - --include=requirements/base.txt - - --include=requirements/zeromq.txt - - requirements/static/ci/pkgtests-windows.in - - id: pip-tools-compile alias: compile-ci-windows-pkg-3.10-requirements name: PKG tests Windows CI Py3.10 Requirements @@ -1071,6 +993,7 @@ repos: - --include=requirements/base.txt - --include=requirements/zeromq.txt - requirements/static/ci/pkgtests-windows.in + # <---- PKG ci requirements----------------------------------------------------------------------------------------- diff --git a/noxfile.py b/noxfile.py index 21d017666d17..85fca93a5bb0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1797,14 +1797,6 @@ def _pkg_test(session, cmd_args, test_type, onedir=False): _pytest(session, False, pytest_args, env=env) -@nox.session(python=_PYTHON_VERSIONS, name="test-pkgs") -def test_pkgs(session): - """ - pytest pkg tests session - """ - _pkg_test(session, ["pkg/tests/"], "pkg") - - @nox.session( python=str(ONEDIR_PYTHON_PATH), name="test-pkgs-onedir", @@ -1820,32 +1812,6 @@ def test_pkgs_onedir(session): _pkg_test(session, ["pkg/tests/"], "pkg", onedir=True) -@nox.session(python=_PYTHON_VERSIONS, name="test-upgrade-pkgs") -@nox.parametrize("classic", [False, True]) -def test_upgrade_pkgs(session, classic): - """ - pytest pkg upgrade tests session - """ - test_type = "pkg_upgrade" - cmd_args = [ - "pkg/tests/upgrade/test_salt_upgrade.py::test_salt_upgrade", - "--upgrade", - "--no-uninstall", - ] - if classic: - cmd_args = cmd_args + ["--classic"] - # Workaround for installing and running classic packages from 3005.1 - # They can only run with importlib-metadata<5.0.0. - subprocess.run(["pip3", "install", "importlib-metadata==4.13.0"], check=False) - try: - _pkg_test(session, cmd_args, test_type) - except nox.command.CommandFailed: - sys.exit(1) - - cmd_args = ["pkg/tests/", "--no-install"] + session.posargs - _pkg_test(session, cmd_args, test_type) - - @nox.session( python=str(ONEDIR_PYTHON_PATH), name="test-upgrade-pkgs-onedir", diff --git a/requirements/static/ci/py3.7/pkgtests-windows.txt b/requirements/static/ci/py3.7/pkgtests-windows.txt deleted file mode 100644 index c3d3ecc6dc73..000000000000 --- a/requirements/static/ci/py3.7/pkgtests-windows.txt +++ /dev/null @@ -1,184 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.7/pkgtests-windows.txt requirements/base.txt requirements/static/ci/pkgtests-windows.in requirements/zeromq.txt -# -attrs==22.2.0 - # via - # pytest - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -certifi==2022.12.7 - # via requests -cffi==1.15.1 - # via clr-loader -charset-normalizer==3.0.1 - # via requests -cheroot==9.0.0 - # via cherrypy -cherrypy==18.8.0 - # via -r requirements/static/ci/pkgtests-windows.in -clr-loader==0.2.5 - # via pythonnet -colorama==0.4.6 - # via pytest -contextvars==2.4 - # via -r requirements/base.txt -distlib==0.3.6 - # via virtualenv -distro==1.8.0 - # via - # -r requirements/base.txt - # pytest-skip-markers -exceptiongroup==1.1.0 - # via pytest -filelock==3.9.0 - # via virtualenv -idna==3.4 - # via requests -immutables==0.19 - # via contextvars -importlib-metadata==6.0.0 - # via - # cheroot - # pluggy - # pytest - # virtualenv -importlib-resources==5.10.2 - # via jaraco.text -inflect==6.0.2 - # via jaraco.text -iniconfig==2.0.0 - # via pytest -jaraco.classes==3.2.3 - # via jaraco.collections -jaraco.collections==3.8.0 - # via cherrypy -jaraco.context==4.3.0 - # via jaraco.text -jaraco.functools==3.5.2 - # via - # cheroot - # jaraco.text - # tempora -jaraco.text==3.11.1 - # via jaraco.collections -jinja2==3.1.2 - # via -r requirements/base.txt -jmespath==1.0.1 - # via -r requirements/base.txt -looseversion==1.0.3 - # via -r requirements/base.txt -markupsafe==2.1.2 - # via - # -r requirements/base.txt - # jinja2 -more-itertools==9.0.0 - # via - # cheroot - # cherrypy - # jaraco.classes - # jaraco.functools - # jaraco.text -msgpack==1.0.4 - # via - # -r requirements/base.txt - # pytest-salt-factories -packaging==23.0 - # via - # -r requirements/base.txt - # pytest -platformdirs==2.6.2 - # via virtualenv -pluggy==1.0.0 - # via pytest -portend==3.1.0 - # via cherrypy -psutil==5.9.4 - # via - # -r requirements/base.txt - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pycparser==2.21 - # via cffi -pycryptodomex==3.17 - # via -r requirements/crypto.txt -pydantic==1.10.4 - # via inflect -pytest-helpers-namespace==2021.12.29 - # via - # pytest-salt-factories - # pytest-shell-utilities -pytest-salt-factories==1.0.0rc17 - # via -r requirements/static/ci/pkgtests-windows.in -pytest-shell-utilities==1.7.0 - # via pytest-salt-factories -pytest-skip-markers==1.4.0 - # via - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pytest-system-statistics==1.0.2 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.2.1 - # via - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics - # pytest-tempdir -pythonnet==3.0.1 ; python_version >= "3.7" - # via -r requirements/static/ci/pkgtests-windows.in -pytz==2022.7.1 - # via tempora -pywin32==305 - # via - # cherrypy - # pytest-skip-markers - # wmi -pyyaml==6.0 - # via -r requirements/base.txt -pyzmq==25.0.0 ; sys_platform == "win32" - # via - # -r requirements/zeromq.txt - # pytest-salt-factories -requests==2.28.2 - # via -r requirements/base.txt -six==1.16.0 - # via cheroot -tempora==5.2.1 - # via portend -tomli==2.0.1 - # via pytest -typing-extensions==4.4.0 - # via - # immutables - # importlib-metadata - # platformdirs - # pydantic - # pytest-shell-utilities - # pytest-system-statistics -urllib3==1.26.14 - # via requests -virtualenv==20.18.0 - # via pytest-salt-factories -wmi==1.5.1 ; sys_platform == "win32" - # via -r requirements/static/ci/pkgtests-windows.in -zc.lockfile==2.0 - # via cherrypy -zipp==3.12.1 - # via - # importlib-metadata - # importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/static/ci/py3.7/pkgtests.txt b/requirements/static/ci/py3.7/pkgtests.txt deleted file mode 100644 index b0b198cab633..000000000000 --- a/requirements/static/ci/py3.7/pkgtests.txt +++ /dev/null @@ -1,166 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.7/pkgtests.txt requirements/base.txt requirements/static/ci/pkgtests.in requirements/zeromq.txt -# -attrs==22.2.0 - # via - # pytest - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -certifi==2022.12.7 - # via requests -charset-normalizer==3.0.1 - # via requests -cheroot==9.0.0 - # via cherrypy -cherrypy==18.8.0 - # via -r requirements/static/ci/pkgtests.in -contextvars==2.4 - # via -r requirements/base.txt -distlib==0.3.6 - # via virtualenv -distro==1.8.0 - # via - # -r requirements/base.txt - # pytest-skip-markers -exceptiongroup==1.1.0 - # via pytest -filelock==3.9.0 - # via virtualenv -idna==3.4 - # via requests -immutables==0.19 - # via contextvars -importlib-metadata==6.0.0 - # via - # cheroot - # pluggy - # pytest - # virtualenv -importlib-resources==5.10.2 - # via jaraco.text -inflect==6.0.2 - # via jaraco.text -iniconfig==2.0.0 - # via pytest -jaraco.classes==3.2.3 - # via jaraco.collections -jaraco.collections==3.8.0 - # via cherrypy -jaraco.context==4.2.0 - # via jaraco.text -jaraco.functools==3.5.2 - # via - # cheroot - # jaraco.text -jaraco.text==3.11.0 - # via jaraco.collections -jinja2==3.1.2 - # via -r requirements/base.txt -jmespath==1.0.1 - # via -r requirements/base.txt -looseversion==1.0.3 - # via -r requirements/base.txt -markupsafe==2.1.1 - # via - # -r requirements/base.txt - # jinja2 -more-itertools==9.0.0 - # via - # cheroot - # cherrypy - # jaraco.classes - # jaraco.functools - # jaraco.text -msgpack==1.0.4 - # via - # -r requirements/base.txt - # pytest-salt-factories -packaging==23.0 - # via - # -r requirements/base.txt - # pytest -platformdirs==2.6.2 - # via virtualenv -pluggy==1.0.0 - # via pytest -portend==3.1.0 - # via cherrypy -psutil==5.9.4 - # via - # -r requirements/base.txt - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pycryptodomex==3.16.0 - # via -r requirements/crypto.txt -pydantic==1.10.4 - # via inflect -pytest-helpers-namespace==2021.12.29 - # via - # pytest-salt-factories - # pytest-shell-utilities -pytest-salt-factories==1.0.0rc17 - # via -r requirements/static/ci/pkgtests.in -pytest-shell-utilities==1.7.0 - # via pytest-salt-factories -pytest-skip-markers==1.4.0 - # via - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pytest-system-statistics==1.0.2 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.2.1 - # via - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics - # pytest-tempdir -pytz==2022.7.1 - # via tempora -pyyaml==6.0 - # via -r requirements/base.txt -pyzmq==22.0.3 - # via - # -r requirements/zeromq.txt - # pytest-salt-factories -requests==2.28.2 - # via -r requirements/base.txt -six==1.16.0 - # via cheroot -tempora==5.2.0 - # via portend -tomli==2.0.1 - # via pytest -typing-extensions==4.4.0 - # via - # immutables - # importlib-metadata - # platformdirs - # pydantic - # pytest-shell-utilities - # pytest-system-statistics -urllib3==1.26.14 - # via requests -virtualenv==20.17.1 - # via pytest-salt-factories -zc.lockfile==2.0 - # via cherrypy -zipp==3.11.0 - # via - # importlib-metadata - # importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/static/ci/py3.8/pkgtests-windows.txt b/requirements/static/ci/py3.8/pkgtests-windows.txt deleted file mode 100644 index 618d69a23554..000000000000 --- a/requirements/static/ci/py3.8/pkgtests-windows.txt +++ /dev/null @@ -1,173 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.8/pkgtests-windows.txt requirements/base.txt requirements/static/ci/pkgtests-windows.in requirements/zeromq.txt -# -attrs==22.2.0 - # via - # pytest - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -certifi==2022.12.7 - # via requests -cffi==1.15.1 - # via clr-loader -charset-normalizer==3.0.1 - # via requests -cheroot==9.0.0 - # via cherrypy -cherrypy==18.8.0 - # via -r requirements/static/ci/pkgtests-windows.in -clr-loader==0.2.5 - # via pythonnet -colorama==0.4.6 - # via pytest -contextvars==2.4 - # via -r requirements/base.txt -distlib==0.3.6 - # via virtualenv -distro==1.8.0 - # via - # -r requirements/base.txt - # pytest-skip-markers -exceptiongroup==1.1.0 - # via pytest -filelock==3.9.0 - # via virtualenv -idna==3.4 - # via requests -immutables==0.19 - # via contextvars -importlib-resources==5.10.2 - # via jaraco.text -inflect==6.0.2 - # via jaraco.text -iniconfig==2.0.0 - # via pytest -jaraco.classes==3.2.3 - # via jaraco.collections -jaraco.collections==3.8.0 - # via cherrypy -jaraco.context==4.3.0 - # via jaraco.text -jaraco.functools==3.5.2 - # via - # cheroot - # jaraco.text - # tempora -jaraco.text==3.11.1 - # via jaraco.collections -jinja2==3.1.2 - # via -r requirements/base.txt -jmespath==1.0.1 - # via -r requirements/base.txt -looseversion==1.0.3 - # via -r requirements/base.txt -markupsafe==2.1.2 - # via - # -r requirements/base.txt - # jinja2 -more-itertools==9.0.0 - # via - # cheroot - # cherrypy - # jaraco.classes - # jaraco.functools - # jaraco.text -msgpack==1.0.4 - # via - # -r requirements/base.txt - # pytest-salt-factories -packaging==23.0 - # via - # -r requirements/base.txt - # pytest -platformdirs==2.6.2 - # via virtualenv -pluggy==1.0.0 - # via pytest -portend==3.1.0 - # via cherrypy -psutil==5.9.4 - # via - # -r requirements/base.txt - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pycparser==2.21 - # via cffi -pycryptodomex==3.17 - # via -r requirements/crypto.txt -pydantic==1.10.4 - # via inflect -pytest-helpers-namespace==2021.12.29 - # via - # pytest-salt-factories - # pytest-shell-utilities -pytest-salt-factories==1.0.0rc17 - # via -r requirements/static/ci/pkgtests-windows.in -pytest-shell-utilities==1.7.0 - # via pytest-salt-factories -pytest-skip-markers==1.4.0 - # via - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pytest-system-statistics==1.0.2 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.2.1 - # via - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics - # pytest-tempdir -pythonnet==3.0.1 ; python_version >= "3.7" - # via -r requirements/static/ci/pkgtests-windows.in -pytz==2022.7.1 - # via tempora -pywin32==305 - # via - # cherrypy - # pytest-skip-markers - # wmi -pyyaml==6.0 - # via -r requirements/base.txt -pyzmq==25.0.0 ; sys_platform == "win32" - # via - # -r requirements/zeromq.txt - # pytest-salt-factories -requests==2.28.2 - # via -r requirements/base.txt -six==1.16.0 - # via cheroot -tempora==5.2.1 - # via portend -tomli==2.0.1 - # via pytest -typing-extensions==4.4.0 - # via - # pydantic - # pytest-shell-utilities - # pytest-system-statistics -urllib3==1.26.14 - # via requests -virtualenv==20.18.0 - # via pytest-salt-factories -wmi==1.5.1 ; sys_platform == "win32" - # via -r requirements/static/ci/pkgtests-windows.in -zc.lockfile==2.0 - # via cherrypy -zipp==3.12.1 - # via importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/static/ci/py3.8/pkgtests.txt b/requirements/static/ci/py3.8/pkgtests.txt deleted file mode 100644 index eef7b8631011..000000000000 --- a/requirements/static/ci/py3.8/pkgtests.txt +++ /dev/null @@ -1,155 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.8/pkgtests.txt requirements/base.txt requirements/static/ci/pkgtests.in requirements/zeromq.txt -# -attrs==22.2.0 - # via - # pytest - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -certifi==2022.12.7 - # via requests -charset-normalizer==3.0.1 - # via requests -cheroot==9.0.0 - # via cherrypy -cherrypy==18.8.0 - # via -r requirements/static/ci/pkgtests.in -contextvars==2.4 - # via -r requirements/base.txt -distlib==0.3.6 - # via virtualenv -distro==1.8.0 - # via - # -r requirements/base.txt - # pytest-skip-markers -exceptiongroup==1.1.0 - # via pytest -filelock==3.9.0 - # via virtualenv -idna==3.4 - # via requests -immutables==0.19 - # via contextvars -importlib-resources==5.10.2 - # via jaraco.text -inflect==6.0.2 - # via jaraco.text -iniconfig==2.0.0 - # via pytest -jaraco.classes==3.2.3 - # via jaraco.collections -jaraco.collections==3.8.0 - # via cherrypy -jaraco.context==4.2.0 - # via jaraco.text -jaraco.functools==3.5.2 - # via - # cheroot - # jaraco.text -jaraco.text==3.11.0 - # via jaraco.collections -jinja2==3.1.2 - # via -r requirements/base.txt -jmespath==1.0.1 - # via -r requirements/base.txt -looseversion==1.0.3 - # via -r requirements/base.txt -markupsafe==2.1.1 - # via - # -r requirements/base.txt - # jinja2 -more-itertools==9.0.0 - # via - # cheroot - # cherrypy - # jaraco.classes - # jaraco.functools - # jaraco.text -msgpack==1.0.4 - # via - # -r requirements/base.txt - # pytest-salt-factories -packaging==23.0 - # via - # -r requirements/base.txt - # pytest -platformdirs==2.6.2 - # via virtualenv -pluggy==1.0.0 - # via pytest -portend==3.1.0 - # via cherrypy -psutil==5.9.4 - # via - # -r requirements/base.txt - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pycryptodomex==3.16.0 - # via -r requirements/crypto.txt -pydantic==1.10.4 - # via inflect -pytest-helpers-namespace==2021.12.29 - # via - # pytest-salt-factories - # pytest-shell-utilities -pytest-salt-factories==1.0.0rc17 - # via -r requirements/static/ci/pkgtests.in -pytest-shell-utilities==1.7.0 - # via pytest-salt-factories -pytest-skip-markers==1.4.0 - # via - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pytest-system-statistics==1.0.2 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.2.1 - # via - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics - # pytest-tempdir -pytz==2022.7.1 - # via tempora -pyyaml==6.0 - # via -r requirements/base.txt -pyzmq==22.0.3 - # via - # -r requirements/zeromq.txt - # pytest-salt-factories -requests==2.28.2 - # via -r requirements/base.txt -six==1.16.0 - # via cheroot -tempora==5.2.0 - # via portend -tomli==2.0.1 - # via pytest -typing-extensions==4.4.0 - # via - # pydantic - # pytest-shell-utilities - # pytest-system-statistics -urllib3==1.26.14 - # via requests -virtualenv==20.17.1 - # via pytest-salt-factories -zc.lockfile==2.0 - # via cherrypy -zipp==3.11.0 - # via importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/static/ci/py3.9/pkgtests-windows.txt b/requirements/static/ci/py3.9/pkgtests-windows.txt deleted file mode 100644 index 54ccbe113dea..000000000000 --- a/requirements/static/ci/py3.9/pkgtests-windows.txt +++ /dev/null @@ -1,169 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.9/pkgtests-windows.txt requirements/base.txt requirements/static/ci/pkgtests-windows.in requirements/zeromq.txt -# -attrs==22.2.0 - # via - # pytest - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -certifi==2022.12.7 - # via requests -cffi==1.15.1 - # via clr-loader -charset-normalizer==3.0.1 - # via requests -cheroot==9.0.0 - # via cherrypy -cherrypy==18.8.0 - # via -r requirements/static/ci/pkgtests-windows.in -clr-loader==0.2.5 - # via pythonnet -colorama==0.4.6 - # via pytest -contextvars==2.4 - # via -r requirements/base.txt -distlib==0.3.6 - # via virtualenv -distro==1.8.0 - # via - # -r requirements/base.txt - # pytest-skip-markers -exceptiongroup==1.1.0 - # via pytest -filelock==3.9.0 - # via virtualenv -idna==3.4 - # via requests -immutables==0.19 - # via contextvars -inflect==6.0.2 - # via jaraco.text -iniconfig==2.0.0 - # via pytest -jaraco.classes==3.2.3 - # via jaraco.collections -jaraco.collections==3.8.0 - # via cherrypy -jaraco.context==4.3.0 - # via jaraco.text -jaraco.functools==3.5.2 - # via - # cheroot - # jaraco.text - # tempora -jaraco.text==3.11.1 - # via jaraco.collections -jinja2==3.1.2 - # via -r requirements/base.txt -jmespath==1.0.1 - # via -r requirements/base.txt -looseversion==1.0.3 - # via -r requirements/base.txt -markupsafe==2.1.2 - # via - # -r requirements/base.txt - # jinja2 -more-itertools==9.0.0 - # via - # cheroot - # cherrypy - # jaraco.classes - # jaraco.functools - # jaraco.text -msgpack==1.0.4 - # via - # -r requirements/base.txt - # pytest-salt-factories -packaging==23.0 - # via - # -r requirements/base.txt - # pytest -platformdirs==2.6.2 - # via virtualenv -pluggy==1.0.0 - # via pytest -portend==3.1.0 - # via cherrypy -psutil==5.9.4 - # via - # -r requirements/base.txt - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pycparser==2.21 - # via cffi -pycryptodomex==3.17 - # via -r requirements/crypto.txt -pydantic==1.10.4 - # via inflect -pytest-helpers-namespace==2021.12.29 - # via - # pytest-salt-factories - # pytest-shell-utilities -pytest-salt-factories==1.0.0rc17 - # via -r requirements/static/ci/pkgtests-windows.in -pytest-shell-utilities==1.7.0 - # via pytest-salt-factories -pytest-skip-markers==1.4.0 - # via - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pytest-system-statistics==1.0.2 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.2.1 - # via - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics - # pytest-tempdir -pythonnet==3.0.1 ; python_version >= "3.7" - # via -r requirements/static/ci/pkgtests-windows.in -pytz==2022.7.1 - # via tempora -pywin32==305 - # via - # cherrypy - # pytest-skip-markers - # wmi -pyyaml==6.0 - # via -r requirements/base.txt -pyzmq==25.0.0 ; sys_platform == "win32" - # via - # -r requirements/zeromq.txt - # pytest-salt-factories -requests==2.28.2 - # via -r requirements/base.txt -six==1.16.0 - # via cheroot -tempora==5.2.1 - # via portend -tomli==2.0.1 - # via pytest -typing-extensions==4.4.0 - # via - # pydantic - # pytest-shell-utilities - # pytest-system-statistics -urllib3==1.26.14 - # via requests -virtualenv==20.18.0 - # via pytest-salt-factories -wmi==1.5.1 ; sys_platform == "win32" - # via -r requirements/static/ci/pkgtests-windows.in -zc.lockfile==2.0 - # via cherrypy - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/static/ci/py3.9/pkgtests.txt b/requirements/static/ci/py3.9/pkgtests.txt deleted file mode 100644 index 6e742ca9dd2c..000000000000 --- a/requirements/static/ci/py3.9/pkgtests.txt +++ /dev/null @@ -1,151 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/static/ci/py3.9/pkgtests.txt requirements/base.txt requirements/static/ci/pkgtests.in requirements/zeromq.txt -# -attrs==22.2.0 - # via - # pytest - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics -autocommand==2.2.2 - # via jaraco.text -certifi==2022.12.7 - # via requests -charset-normalizer==3.0.1 - # via requests -cheroot==9.0.0 - # via cherrypy -cherrypy==18.8.0 - # via -r requirements/static/ci/pkgtests.in -contextvars==2.4 - # via -r requirements/base.txt -distlib==0.3.6 - # via virtualenv -distro==1.8.0 - # via - # -r requirements/base.txt - # pytest-skip-markers -exceptiongroup==1.1.0 - # via pytest -filelock==3.9.0 - # via virtualenv -idna==3.4 - # via requests -immutables==0.19 - # via contextvars -inflect==6.0.2 - # via jaraco.text -iniconfig==2.0.0 - # via pytest -jaraco.classes==3.2.3 - # via jaraco.collections -jaraco.collections==3.8.0 - # via cherrypy -jaraco.context==4.2.0 - # via jaraco.text -jaraco.functools==3.5.2 - # via - # cheroot - # jaraco.text -jaraco.text==3.11.0 - # via jaraco.collections -jinja2==3.1.2 - # via -r requirements/base.txt -jmespath==1.0.1 - # via -r requirements/base.txt -looseversion==1.0.3 - # via -r requirements/base.txt -markupsafe==2.1.1 - # via - # -r requirements/base.txt - # jinja2 -more-itertools==9.0.0 - # via - # cheroot - # cherrypy - # jaraco.classes - # jaraco.functools - # jaraco.text -msgpack==1.0.4 - # via - # -r requirements/base.txt - # pytest-salt-factories -packaging==23.0 - # via - # -r requirements/base.txt - # pytest -platformdirs==2.6.2 - # via virtualenv -pluggy==1.0.0 - # via pytest -portend==3.1.0 - # via cherrypy -psutil==5.9.4 - # via - # -r requirements/base.txt - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pycryptodomex==3.16.0 - # via -r requirements/crypto.txt -pydantic==1.10.4 - # via inflect -pytest-helpers-namespace==2021.12.29 - # via - # pytest-salt-factories - # pytest-shell-utilities -pytest-salt-factories==1.0.0rc17 - # via -r requirements/static/ci/pkgtests.in -pytest-shell-utilities==1.7.0 - # via pytest-salt-factories -pytest-skip-markers==1.4.0 - # via - # pytest-salt-factories - # pytest-shell-utilities - # pytest-system-statistics -pytest-system-statistics==1.0.2 - # via pytest-salt-factories -pytest-tempdir==2019.10.12 - # via pytest-salt-factories -pytest==7.2.1 - # via - # pytest-helpers-namespace - # pytest-salt-factories - # pytest-shell-utilities - # pytest-skip-markers - # pytest-system-statistics - # pytest-tempdir -pytz==2022.7.1 - # via tempora -pyyaml==6.0 - # via -r requirements/base.txt -pyzmq==25.0.0 - # via - # -r requirements/zeromq.txt - # pytest-salt-factories -requests==2.28.2 - # via -r requirements/base.txt -six==1.16.0 - # via cheroot -tempora==5.2.0 - # via portend -tomli==2.0.1 - # via pytest -typing-extensions==4.4.0 - # via - # pydantic - # pytest-shell-utilities - # pytest-system-statistics -urllib3==1.26.14 - # via requests -virtualenv==20.17.1 - # via pytest-salt-factories -zc.lockfile==2.0 - # via cherrypy - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From 8332091303d3187f8fcb365689314b3c79d38c6a Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 8 Mar 2023 18:01:43 +0000 Subject: [PATCH 6/7] Fix the bash comparisson Signed-off-by: Pedro Algarvio --- .github/workflows/nightly.yml | 2 +- .github/workflows/scheduled.yml | 2 +- .github/workflows/templates/nightly.yml.jinja | 2 +- .github/workflows/templates/scheduled.yml.jinja | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 31b0c4ba9e14..bf3780a75197 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -34,7 +34,7 @@ jobs: - name: Check Requirements id: check-requirements run: | - if [ "${RUN_SCHEDULED_BUILDS}" -eq 1 ]; then + if [ "${RUN_SCHEDULED_BUILDS}" = "1" ]; then echo "Running workflow because RUN_SCHEDULED_BUILDS=1" echo "requirements-met=true" >> "${GITHUB_OUTPUT}" elif [ "${{ github.event.repository.fork }}" = "true" ]; then diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index c0571835dc7b..97d239993785 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -32,7 +32,7 @@ jobs: - name: Check Requirements id: check-requirements run: | - if [ "${RUN_SCHEDULED_BUILDS}" -eq 1 ]; then + if [ "${RUN_SCHEDULED_BUILDS}" = "1" ]; then echo "Running workflow because RUN_SCHEDULED_BUILDS=1" echo "requirements-met=true" >> "${GITHUB_OUTPUT}" elif [ "${{ github.event.repository.fork }}" = "true" ]; then diff --git a/.github/workflows/templates/nightly.yml.jinja b/.github/workflows/templates/nightly.yml.jinja index 4c00e5915337..7ab5f2de1306 100644 --- a/.github/workflows/templates/nightly.yml.jinja +++ b/.github/workflows/templates/nightly.yml.jinja @@ -41,7 +41,7 @@ concurrency: - name: Check Requirements id: check-requirements run: | - if [ "${RUN_SCHEDULED_BUILDS}" -eq 1 ]; then + if [ "${RUN_SCHEDULED_BUILDS}" = "1" ]; then echo "Running workflow because RUN_SCHEDULED_BUILDS=1" echo "requirements-met=true" >> "${GITHUB_OUTPUT}" elif [ "${{ github.event.repository.fork }}" = "true" ]; then diff --git a/.github/workflows/templates/scheduled.yml.jinja b/.github/workflows/templates/scheduled.yml.jinja index 028062f58326..96c8d160a7c2 100644 --- a/.github/workflows/templates/scheduled.yml.jinja +++ b/.github/workflows/templates/scheduled.yml.jinja @@ -33,7 +33,7 @@ concurrency: - name: Check Requirements id: check-requirements run: | - if [ "${RUN_SCHEDULED_BUILDS}" -eq 1 ]; then + if [ "${RUN_SCHEDULED_BUILDS}" = "1" ]; then echo "Running workflow because RUN_SCHEDULED_BUILDS=1" echo "requirements-met=true" >> "${GITHUB_OUTPUT}" elif [ "${{ github.event.repository.fork }}" = "true" ]; then From bbb3ca4c8e9db912806930c6ef29eaa3af663576 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 8 Mar 2023 19:48:56 +0000 Subject: [PATCH 7/7] Use `requests` to download files Signed-off-by: Pedro Algarvio --- pkg/tests/support/helpers.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pkg/tests/support/helpers.py b/pkg/tests/support/helpers.py index f9353c52bbee..ac68f79c1a8d 100644 --- a/pkg/tests/support/helpers.py +++ b/pkg/tests/support/helpers.py @@ -654,14 +654,10 @@ def install_previous(self): f"https://repo.saltproject.io/{root_url}{distro_name}/{self.distro_version}/{arch}/{major_ver}/{gpg_key}", ) self._check_retcode(ret) - ret = self.proc.run( - "curl", - "-fsSL", + download_file( f"https://repo.saltproject.io/{root_url}{distro_name}/{self.distro_version}/{arch}/{major_ver}.repo", - "-o", f"/etc/yum.repos.d/salt-{distro_name}.repo", ) - self._check_retcode(ret) ret = self.proc.run(self.pkg_mngr, "clean", "expire-cache") self._check_retcode(ret) ret = self.proc.run( @@ -685,14 +681,10 @@ def install_previous(self): else: arch = "amd64" pathlib.Path("/etc/apt/keyrings").mkdir(parents=True, exist_ok=True) - ret = self.proc.run( - "curl", - "-fsSL", - "-o", - "/etc/apt/keyrings/salt-archive-keyring.gpg", + download_file( f"https://repo.saltproject.io/{root_url}{distro_name}/{self.distro_version}/{arch}/{major_ver}/salt-archive-keyring.gpg", + "/etc/apt/keyrings/salt-archive-keyring.gpg", ) - self._check_retcode(ret) with open( pathlib.Path("/etc", "apt", "sources.list.d", "salt.list"), "w" ) as fp: @@ -763,14 +755,10 @@ def install_previous(self): mac_pkg_url = f"https://repo.saltproject.io/salt/py3/macos/{major_ver}.{minor_ver}-1/{mac_pkg}" mac_pkg_path = f"/tmp/{mac_pkg}" if not os.path.exists(mac_pkg_path): - ret = self.proc.run( - "curl", - "-fsSL", - "-o", - f"/tmp/{mac_pkg}", + download_file( f"{mac_pkg_url}", + f"/tmp/{mac_pkg}", ) - self._check_retcode(ret) ret = self.proc.run("installer", "-pkg", mac_pkg_path, "-target", "/") self._check_retcode(ret) @@ -1700,3 +1688,13 @@ def remove_stale_master_key(master): master.id, key_path, ) + + +def download_file(url, dest): + # NOTE the stream=True parameter below + with requests.get(url, stream=True) as r: + r.raise_for_status() + with open(dest, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + if chunk: + f.write(chunk)