diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index bad75f3b..afe0b118 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -9,12 +9,17 @@ ADD ./{{ item.name }} ./{{ item.name }} # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads # Hacking botocore (https://github.com/boto/botocore/pull/1990) -RUN pip3 install +RUN pip3 install --upgrade pip setuptools wheel && \ + rm -rf /root/.cache/pip && \ + pip3 install {%- if s3_test | default(false) -%} {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} {%- for item in plugins -%} {{ " " }}{{ item.source }} +{%- if item.upperbounds | default(false) -%} +{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt +{%- endif -%} {%- if item.lowerbounds | default(false) -%} {{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt {%- endif -%} @@ -22,7 +27,8 @@ RUN pip3 install {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ + rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py new file mode 100755 index 00000000..3372754e --- /dev/null +++ b/.ci/scripts/calc_constraints.py @@ -0,0 +1,118 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_ostree' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import fileinput +import urllib.request +import sys +from packaging.requirements import Requirement +from packaging.version import Version +import yaml + + +CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" + + +def fetch_pulpcore_upper_bound(requirement): + with urllib.request.urlopen(CORE_TEMPLATE_URL) as f: + template = yaml.safe_load(f.read()) + supported_versions = template["supported_release_branches"] + supported_versions.append(template["latest_release_branch"]) + applicable_versions = sorted( + requirement.specifier.filter((Version(v) for v in supported_versions)) + ) + if len(applicable_versions) == 0: + raise Exception("No supported pulpcore version in required range.") + return f"{requirement.name}~={applicable_versions[-1]}" + + +def split_comment(line): + split_line = line.split("#", maxsplit=1) + try: + comment = " # " + split_line[1].strip() + except IndexError: + comment = "" + return split_line[0].strip(), comment + + +def to_upper_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + if requirement.name == "pulpcore": + # An exception to allow for pulpcore deprecation policy. + return fetch_pulpcore_upper_bound(requirement) + for spec in requirement.specifier: + if spec.operator == "~=": + return f"# NO BETTER CONSTRAINT: {req}" + if spec.operator == "<=": + operator = "==" + max_version = spec.version + return f"{requirement.name}{operator}{max_version}" + if spec.operator == "<": + operator = "~=" + version = Version(spec.version) + if version.micro != 0: + max_version = f"{version.major}.{version.minor}.{version.micro-1}" + elif version.minor != 0: + max_version = f"{version.major}.{version.minor-1}" + elif version.major != 0: + max_version = f"{version.major-1}.0" + else: + return f"# NO BETTER CONSTRAINT: {req}" + return f"{requirement.name}{operator}{max_version}" + return f"# NO UPPER BOUND: {req}" + + +def to_lower_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + for spec in requirement.specifier: + if spec.operator == ">=": + if requirement.name == "pulpcore": + # Currently an exception to allow for pulpcore bugfix releases. + # TODO Semver libraries should be allowed too. + operator = "~=" + else: + operator = "==" + min_version = spec.version + return f"{requirement.name}{operator}{min_version}" + return f"# NO LOWER BOUND: {req}" + + +def main(): + """Calculate constraints for the lower bound of dependencies where possible.""" + parser = argparse.ArgumentParser( + prog=sys.argv[0], + description="Calculate constraints for the lower or upper bound of dependencies where " + "possible.", + ) + parser.add_argument("-u", "--upper", action="store_true") + parser.add_argument("filename", nargs="*") + args = parser.parse_args() + + with fileinput.input(files=args.filename) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) + else: + req, comment = split_comment(line) + if args.upper: + new_req = to_upper_bound(req) + else: + new_req = to_lower_bound(req) + print(new_req + comment) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/calc_deps_lowerbounds.py b/.ci/scripts/calc_deps_lowerbounds.py deleted file mode 100755 index 599006ee..00000000 --- a/.ci/scripts/calc_deps_lowerbounds.py +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_ostree' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -from packaging.requirements import Requirement - - -def main(): - """Calculate the lower bound of dependencies where possible.""" - with open("requirements.txt") as req_file: - for line in req_file: - try: - requirement = Requirement(line) - except ValueError: - print(line.strip()) - else: - for spec in requirement.specifier: - if spec.operator == ">=": - if requirement.name == "pulpcore": - operator = "~=" - else: - operator = "==" - min_version = str(spec)[2:] - print(f"{requirement.name}{operator}{min_version}") - break - else: - print(line.strip()) - - -if __name__ == "__main__": - main() diff --git a/.github/template_gitref b/.github/template_gitref index b222a637..9e5cdc02 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-351-g4e8a1a7 +2021.08.26-364-g6f9579c diff --git a/.github/workflows/kanban.yml b/.github/workflows/kanban.yml deleted file mode 100644 index 09554117..00000000 --- a/.github/workflows/kanban.yml +++ /dev/null @@ -1,103 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_ostree' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -# Manage issues in a project board using https://github.com/leonsteinhaeuser/project-beta-automations - ---- -name: Kanban -on: - pull_request_target: - issues: - types: - - labeled - - reopened - - assigned - - closed - -env: - free_to_take: Free to take - in_progress: In Progress - needs_review: Needs review - done: Done - -jobs: - # only prio-list labeled items should be added to the board - add-to-project-board: - if: github.event_name == 'issues' && contains(github.event.issue.labels.*.name, 'prio-list') && contains(fromJson('["labeled", "reopened"]'), github.event.action) - runs-on: ubuntu-latest - steps: - - name: Add issue to Free-to-take list - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.free_to_take }} # Target status - - move-to-inprogress: - if: github.event_name == 'issues' && github.event.action == 'assigned' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the In Progress column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.in_progress }} # Target status - - find-linked-issues: - if: github.event_name == 'pull_request_target' - runs-on: ubuntu-latest - name: Find issues linked to a PR - outputs: - linked-issues: ${{ steps.linked-issues.outputs.issues }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Get Linked Issues Action - uses: kin/gh-action-get-linked-issues@v1.0 - id: linked-issues - with: - access-token: ${{ secrets.RELEASE_TOKEN }} - - move-to-needs-review: - if: github.event_name == 'pull_request_target' && contains(fromJson(needs.find-linked-issues.outputs.linked-issues).*.issue.state, 'open') - runs-on: ubuntu-latest - name: Move linked issues to Needs Review - needs: find-linked-issues - strategy: - max-parallel: 3 - matrix: - issues: ${{ fromJSON(needs.find-linked-issues.outputs.linked-issues) }} - steps: - - name: Move to Needs Review - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ matrix.issues.issue.node_id }} - operation_mode: status - status_value: ${{ env.needs_review }} # Target status - - move-to-done: - if: github.event_name == 'issues' && github.event.action == 'closed' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the Done column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.done }} # Target status diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 17715ef4..936ea91a 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -65,9 +65,11 @@ then exit $s fi +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt +fi if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt - sed -i 's/\[.*\]//g' lowerbounds_constraints.txt + python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh index 9fcf2ffb..72c616f4 100755 --- a/.github/workflows/scripts/build_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -18,9 +18,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pushd ../pulp-openapi-generator rm -rf "pulp_ostree-client" -# We need to copy that over to be visible in the container. -cp "../pulp_ostree/ostree-api.json" . -./gen-client.sh "ostree-api.json" "ostree" python "pulp_ostree" +./gen-client.sh "../pulp_ostree/ostree-api.json" "ostree" python "pulp_ostree" pushd pulp_ostree-client python setup.py sdist bdist_wheel --python-tag py3 diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh index 18f7dd72..a5856333 100755 --- a/.github/workflows/scripts/build_ruby_client.sh +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -18,15 +18,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pushd ../pulp-openapi-generator rm -rf "pulp_ostree-client" -# We need to copy that over to be visible in the container. -#cp "../pulp_ostree/ostree-api.json" . -#./gen-client.sh "ostree-api.json" "ostree" ruby "pulp_ostree" - -# ------------- -# The generator still needs to have it called api.json at this time... -cp "../pulp_ostree/api.json" . -./gen-client.sh "api.json" "ostree" ruby "pulp_ostree" -# ------------- +./gen-client.sh "../pulp_ostree/ostree-api.json" "ostree" ruby "pulp_ostree" pushd pulp_ostree-client gem build pulp_ostree_client diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index b99832a0..329a40a8 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -55,6 +55,11 @@ if [[ -f ../../ci_requirements.txt ]]; then ci_requirements: true VARSYAML fi +if [ "$TEST" = "pulp" ]; then + cat >> vars/main.yaml << VARSYAML + upperbounds: true +VARSYAML +fi if [ "$TEST" = "lowerbounds" ]; then cat >> vars/main.yaml << VARSYAML lowerbounds: true diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 48e72129..2fe47b50 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -11,8 +11,8 @@ name: "Ostree CI Update" on: schedule: # * is a special character in YAML so you have to quote this string - # runs at 2:30 UTC every Sunday - - cron: '30 2 * * 0' + # runs at 10:30 UTC every Sunday + - cron: '30 10 * * 0' workflow_dispatch: jobs: @@ -79,7 +79,7 @@ jobs: - name: "Run update" working-directory: "pulp_ostree" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" diff --git a/docs/template_gitref b/docs/template_gitref index b222a637..9e5cdc02 100644 --- a/docs/template_gitref +++ b/docs/template_gitref @@ -1 +1 @@ -2021.08.26-351-g4e8a1a7 +2021.08.26-364-g6f9579c diff --git a/template_config.yml b/template_config.yml index d9feaaf1..8b20761a 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-351-g4e8a1a7 +# generated with plugin_template@2021.08.26-364-g6f9579c api_root: /pulp/ black: true @@ -27,7 +27,6 @@ flake8: true flake8_ignore: [] github_org: pulp issue_tracker: github -kanban: true latest_release_branch: '2.3' lint_requirements: true noissue_marker: '[noissue]'