From f15e14e6bc15a18e10cf5c79eb783b74b8482a2f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 15:35:08 -0500 Subject: [PATCH 01/16] Try building and uploading an sdist as well --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7fb9cbb3..ec96ba78 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,12 +59,12 @@ commands: # to rely on whatever resolution logic setuptools would apply. << parameters.python >> -m pip install --upgrade milksnake setuptools_scm # And get these so we can build, fix, and upload a wheel. - << parameters.python >> -m pip install wheel auditwheel twine + << parameters.python >> -m pip install build auditwheel twine - run: - name: "Build Wheel" + name: "Build Packages" command: | - << parameters.python >> -m pip wheel --no-deps . + << parameters.python >> -m build --sdist --wheel - when: condition: << parameters.audit-wheel >> @@ -116,7 +116,7 @@ commands: repo="testpypi" TWINE_PASSWORD="$TWINE_TESTPYPI_PASSWORD" fi - << parameters.python >> -m twine upload --repository $repo python_challenge_bypass_ristretto*.whl + << parameters.python >> -m twine upload --repository $repo python_challenge_bypass_ristretto*.{gz,whl} # Define the actual jobs that will be available to run in a workflow. jobs: From af4b35cb22bfc336424496989505f98c6fa33010 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 15:40:20 -0500 Subject: [PATCH 02/16] [ci debugging] what files did we get --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec96ba78..89ee6543 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,6 +75,7 @@ commands: # Since both macOS and Linux jobs re-use this step, make # sure we only try to use auditwheel on the appropriate # platform. That is, only on Linux. + find << parameters.python >> -m auditwheel repair python_challenge_bypass_ristretto*.whl # Delete the original, unfixed wheel. rm *.whl From d24d2e5d32cde9ce0771fc8c35c0f603fe4b0b9d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 15:43:41 -0500 Subject: [PATCH 03/16] There it is. --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89ee6543..1310d758 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,8 +75,7 @@ commands: # Since both macOS and Linux jobs re-use this step, make # sure we only try to use auditwheel on the appropriate # platform. That is, only on Linux. - find - << parameters.python >> -m auditwheel repair python_challenge_bypass_ristretto*.whl + << parameters.python >> -m auditwheel repair dist/python_challenge_bypass_ristretto*.whl # Delete the original, unfixed wheel. rm *.whl # Move the fixed wheel here for consistency with the From 4429d903c3334507b698a7d25d30d5b79a4c286a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 15:47:19 -0500 Subject: [PATCH 04/16] There is no .whl here anymore --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1310d758..a81712c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,8 +76,6 @@ commands: # sure we only try to use auditwheel on the appropriate # platform. That is, only on Linux. << parameters.python >> -m auditwheel repair dist/python_challenge_bypass_ristretto*.whl - # Delete the original, unfixed wheel. - rm *.whl # Move the fixed wheel here for consistency with the # non-manylinux case. mv wheelhouse/python_challenge_bypass_ristretto*.whl ./ From 725669e13df864e10dc91bde97ec1618de50a08c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 15:47:26 -0500 Subject: [PATCH 05/16] Probably no other .gz but let's just be more precise anyway --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a81712c0..488e0f58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -114,7 +114,7 @@ commands: repo="testpypi" TWINE_PASSWORD="$TWINE_TESTPYPI_PASSWORD" fi - << parameters.python >> -m twine upload --repository $repo python_challenge_bypass_ristretto*.{gz,whl} + << parameters.python >> -m twine upload --repository $repo python_challenge_bypass_ristretto*.{tar.gz,whl} # Define the actual jobs that will be available to run in a workflow. jobs: From c4b55dc33871e893e39c1eff74873418b97b6d1e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 15:51:13 -0500 Subject: [PATCH 06/16] Get the filename pattern right for the sdist Of _course_ they don't follow the same naming rules... --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 488e0f58..3ca194d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -114,7 +114,9 @@ commands: repo="testpypi" TWINE_PASSWORD="$TWINE_TESTPYPI_PASSWORD" fi - << parameters.python >> -m twine upload --repository $repo python_challenge_bypass_ristretto*.{tar.gz,whl} + << parameters.python >> -m twine upload --repository $repo \ + python_challenge_bypass_ristretto*.whl \ + python-challenge-bypass-ristretto*.tar.gz # Define the actual jobs that will be available to run in a workflow. jobs: From 16f5f8aba2ce12006002776f28044e47d9837f6f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 15:53:59 -0500 Subject: [PATCH 07/16] [ci testing] ... where is it? --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ca194d2..ab8ae677 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -114,6 +114,7 @@ commands: repo="testpypi" TWINE_PASSWORD="$TWINE_TESTPYPI_PASSWORD" fi + find << parameters.python >> -m twine upload --repository $repo \ python_challenge_bypass_ristretto*.whl \ python-challenge-bypass-ristretto*.tar.gz From 33a76d7aed56964a30a8127e9b51864832124522 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 15:58:13 -0500 Subject: [PATCH 08/16] It's all in dist/ --- .circleci/config.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab8ae677..2683a77e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,9 +76,13 @@ commands: # sure we only try to use auditwheel on the appropriate # platform. That is, only on Linux. << parameters.python >> -m auditwheel repair dist/python_challenge_bypass_ristretto*.whl - # Move the fixed wheel here for consistency with the - # non-manylinux case. - mv wheelhouse/python_challenge_bypass_ristretto*.whl ./ + + # Get rid of the original wheel. + rm dist/python_challenge_bypass_ristretto*.whl + + # Move the fixed wheel to the original location for + # consistency with the non-manylinux case. + mv wheelhouse/python_challenge_bypass_ristretto*.whl ./dist/ - run: name: "Upload Wheel" @@ -116,8 +120,8 @@ commands: fi find << parameters.python >> -m twine upload --repository $repo \ - python_challenge_bypass_ristretto*.whl \ - python-challenge-bypass-ristretto*.tar.gz + dist/python_challenge_bypass_ristretto*.whl \ + dist/python-challenge-bypass-ristretto*.tar.gz # Define the actual jobs that will be available to run in a workflow. jobs: From 0e4fe67f3e33673becfb970bd985ba396c267d1b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 16:00:36 -0500 Subject: [PATCH 09/16] Remove debug helper --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2683a77e..8b00cb30 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,6 @@ commands: repo="testpypi" TWINE_PASSWORD="$TWINE_TESTPYPI_PASSWORD" fi - find << parameters.python >> -m twine upload --repository $repo \ dist/python_challenge_bypass_ristretto*.whl \ dist/python-challenge-bypass-ristretto*.tar.gz From 997b9c19e536eae4fcc9b4c5a419025ac23386d6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 16:16:00 -0500 Subject: [PATCH 10/16] Try to build Linux packages for tags too --- .circleci/config.yml | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b00cb30..fb14e5f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -278,12 +278,27 @@ workflows: # fails if we don't have any ssh client. pre-command: "yum install -y openssh-clients" python: "/opt/python/cp37-cp37m/bin/python" + filters: &FILTERS + # CircleCI does not run workflows for tags unless you explicitly + # specify tag filters. Additionally, if a job requires any other + # jobs (directly or indirectly), you must specify tag filters for + # those jobs. + # https://circleci.com/docs/2.0/configuration-reference/?section=configuration#tags + branches: + # Keep running on all branches + only: "/.*/" + tags: + # Run on all release tags too + only: "/v.*/" + - "package-manylinux": name: "package-manylinux_2_24-x86_64" executor: "manylinux_2_24-x86_64" # Similar to the manylinux-2014_x86_64 case. pre-command: "apt-get update -y && apt-get install -y openssh-client" python: "/opt/python/cp37-cp37m/bin/python" + filters: + <<: *FILTERS - "package-macos": matrix: @@ -297,14 +312,4 @@ workflows: # A macOS 12.x - "14.1.0" filters: - # CircleCI does not run workflows for tags unless you explicitly - # specify tag filters. Additionally, if a job requires any other - # jobs (directly or indirectly), you must specify tag filters for - # those jobs. - # https://circleci.com/docs/2.0/configuration-reference/?section=configuration#tags - branches: - # Keep running on all branches - only: "/.*/" - tags: - # Run on all release tags too - only: "/v.*/" + <<: *FILTERS From d20b1711ccfbc1f1d0b0258628e32408738b6667 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 16:40:43 -0500 Subject: [PATCH 11/16] Build the packages using the correct version of Python --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 71c6a369..eb6b0333 100644 --- a/flake.nix +++ b/flake.nix @@ -79,8 +79,8 @@ }; python38-challenge-bypass-ristretto = py-module pkgs.python38.pkgs; - python39-challenge-bypass-ristretto = py-module pkgs.python38.pkgs; - python310-challenge-bypass-ristretto = py-module pkgs.python38.pkgs; + python39-challenge-bypass-ristretto = py-module pkgs.python39.pkgs; + python310-challenge-bypass-ristretto = py-module pkgs.python310.pkgs; }; # Define our cross-compiled packages. This currently does not include From 83ea53932a20dd42225b6eb1a090ecf60bccb173 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 16:41:08 -0500 Subject: [PATCH 12/16] Bump the Nix package version --- nix/python-challenge-bypass-ristretto.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/python-challenge-bypass-ristretto.nix b/nix/python-challenge-bypass-ristretto.nix index 7111e7cb..f5ca3d2f 100644 --- a/nix/python-challenge-bypass-ristretto.nix +++ b/nix/python-challenge-bypass-ristretto.nix @@ -2,7 +2,7 @@ # Ristretto implementation. { libchallenge_bypass_ristretto_ffi, python, pythonPackages, milksnake, cffi, attrs, testtools, hypothesis }: pythonPackages.buildPythonPackage rec { - version = "2022.6.30"; + version = "2023.1.23"; pname = "python-challenge-bypass-ristretto"; name = "${pname}-${version}"; src = ../.; From 1a86ccb26f7aaabbe49e9e7917ad0d19f643e8ae Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 16:41:13 -0500 Subject: [PATCH 13/16] write the Nix package version to the Python metadata Giving the Python package the correct metadata, too --- nix/python-challenge-bypass-ristretto.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nix/python-challenge-bypass-ristretto.nix b/nix/python-challenge-bypass-ristretto.nix index f5ca3d2f..0d4a02a4 100644 --- a/nix/python-challenge-bypass-ristretto.nix +++ b/nix/python-challenge-bypass-ristretto.nix @@ -18,6 +18,11 @@ pythonPackages.buildPythonPackage rec { --replace "target/release" "${libchallenge_bypass_ristretto_ffi}/lib" \ --replace "./src" "${libchallenge_bypass_ristretto_ffi.src}/src" \ --replace "'setuptools_scm'" "" + + cat >$sourceRoot/setup.cfg < Date: Mon, 23 Jan 2023 16:53:37 -0500 Subject: [PATCH 14/16] only upload the sdist from one job --- .circleci/config.yml | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fb14e5f1..caa3ecd8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,8 +14,8 @@ executors: # Define some custom commands that we can use as elements of `steps` in job # definitions. commands: - build-wheel: - description: "Build a Python wheel" + build-packages: + description: "Build Python packages" parameters: python: @@ -27,6 +27,10 @@ commands: description: "A boolean controlling whether the auditwheel tool is used to fix up the wheel" type: "boolean" + upload-sdist: + description: "True to upload sdist to the package index, False otherwise" + type: "boolean" + steps: - "checkout" - run: @@ -85,7 +89,7 @@ commands: mv wheelhouse/python_challenge_bypass_ristretto*.whl ./dist/ - run: - name: "Upload Wheel" + name: "Upload Packages" environment: # PyPI authentication configuration for twine so we can upload # packages. TWINE_PASSWORD is set in the CircleCI private @@ -118,9 +122,16 @@ commands: repo="testpypi" TWINE_PASSWORD="$TWINE_TESTPYPI_PASSWORD" fi - << parameters.python >> -m twine upload --repository $repo \ - dist/python_challenge_bypass_ristretto*.whl \ - dist/python-challenge-bypass-ristretto*.tar.gz + + upload() { + << parameters.python >> -m twine upload --repository $repo "$@" + } + + upload dist/python_challenge_bypass_ristretto*.whl + + if << parameters.upload-sdist >>; then + upload dist/python-challenge-bypass-ristretto*.tar.gz + fi # Define the actual jobs that will be available to run in a workflow. jobs: @@ -144,6 +155,9 @@ jobs: description: "the path to the Python executable to use" type: "string" + upload-sdist: + description: "See build-packages" + type: "boolean" executor: "<< parameters.executor >>" @@ -153,9 +167,10 @@ jobs: command: | << parameters.pre-command >> - - "build-wheel": + - "build-packages": python: "<< parameters.python >>" audit-wheel: true + upload-sdist: << parameters.upload-sdist >> package-macos: parameters: @@ -166,8 +181,9 @@ jobs: xcode: "<< parameters.xcode-version >>" steps: - - "build-wheel": + - "build-packages": audit-wheel: false + upload-sdist: false nix-template: &NIX_TEMPLATE @@ -278,6 +294,8 @@ workflows: # fails if we don't have any ssh client. pre-command: "yum install -y openssh-clients" python: "/opt/python/cp37-cp37m/bin/python" + # This one can upload the sdist. + upload-sdist: true filters: &FILTERS # CircleCI does not run workflows for tags unless you explicitly # specify tag filters. Additionally, if a job requires any other @@ -297,6 +315,8 @@ workflows: # Similar to the manylinux-2014_x86_64 case. pre-command: "apt-get update -y && apt-get install -y openssh-client" python: "/opt/python/cp37-cp37m/bin/python" + # The other one can upload the sdist. + upload-sdist: false filters: <<: *FILTERS From 2f3cfa7eda9c30a2a9cda9b4d454f6e7b70c9b2d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 17:00:56 -0500 Subject: [PATCH 15/16] slightly less diff churn --- .circleci/config.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index caa3ecd8..8e4146d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,6 +27,9 @@ commands: description: "A boolean controlling whether the auditwheel tool is used to fix up the wheel" type: "boolean" + # A parameter that lets us direct only one of the jobs to upload the + # sdist. Otherwise twine fails when all jobs except the first try to + # re-upload a package file that already exists. upload-sdist: description: "True to upload sdist to the package index, False otherwise" type: "boolean" @@ -81,11 +84,11 @@ commands: # platform. That is, only on Linux. << parameters.python >> -m auditwheel repair dist/python_challenge_bypass_ristretto*.whl - # Get rid of the original wheel. + # Delete the original, unfixed wheel. rm dist/python_challenge_bypass_ristretto*.whl - # Move the fixed wheel to the original location for - # consistency with the non-manylinux case. + # Move the fixed wheel here for consistency with the + # non-manylinux case. mv wheelhouse/python_challenge_bypass_ristretto*.whl ./dist/ - run: From 10db234c79b5c9a14c5be951983f8512671b9c17 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 2 Mar 2023 16:28:29 -0500 Subject: [PATCH 16/16] Fix shell script syntax --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index fb9710bd..195c3237 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -146,6 +146,7 @@ commands: if << parameters.upload-sdist >>; then upload dist/python-challenge-bypass-ristretto*.tar.gz + fi else echo "Package index credentials unavailable; skipping upload." fi