From 799dcf35ecfcd8a483af3719ca78f1e53c0c6a53 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 23 Oct 2024 12:52:02 -0400 Subject: [PATCH 1/6] build: Test on ubuntu 24.04 --- .github/workflows/check-consistent-dependencies.yml | 2 +- .github/workflows/ci-static-analysis.yml | 2 +- .github/workflows/migrations-check.yml | 4 ++-- .github/workflows/pylint-checks.yml | 4 ++-- .github/workflows/quality-checks.yml | 2 +- .github/workflows/static-assets-check.yml | 2 +- .github/workflows/unit-tests.yml | 12 ++++++------ 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml index 048cbe6b006b..82298af70a54 100644 --- a/.github/workflows/check-consistent-dependencies.yml +++ b/.github/workflows/check-consistent-dependencies.yml @@ -15,7 +15,7 @@ defaults: jobs: check-requirements: name: Compile requirements - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: # Only run remaining steps if there are changes to requirements/** diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index 458e00fc6b1f..d989ff9db288 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -10,7 +10,7 @@ jobs: matrix: python-version: - "3.11" - os: ["ubuntu-22.04"] + os: ["ubuntu-24.04"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 624caddd5309..84e334d68872 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] python-version: - "3.11" # 'pinned' is used to install the latest patch version of Django @@ -126,7 +126,7 @@ jobs: if: always() needs: - check_migrations - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Decide whether the needed jobs succeeded or failed # uses: re-actors/alls-green@v1.2.1 diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 8860aced7f92..9a654e09e711 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -8,7 +8,7 @@ on: jobs: run-pylint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -75,7 +75,7 @@ jobs: if: always() needs: - run-pylint - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Decide whether the needed jobs succeeded or failed # uses: re-actors/alls-green@v1.2.1 diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 84610123493c..310f9f83bf3d 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] python-version: - "3.11" node-version: [20] diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 4fe66e2a7778..e08b2dce8127 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] python-version: - "3.11" node-version: [18, 20] diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 854677b93cff..03d3ce60e24b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -15,7 +15,7 @@ concurrency: jobs: run-tests: name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: python-version: @@ -164,7 +164,7 @@ jobs: overwrite: true collect-and-verify: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Setup Python @@ -229,7 +229,7 @@ jobs: # https://github.com/orgs/community/discussions/33579 success: name: Unit tests successful - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: always() needs: [run-tests] steps: @@ -240,7 +240,7 @@ jobs: jobs: ${{ toJSON(needs) }} compile-warnings-report: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [run-tests] steps: - uses: actions/checkout@v4 @@ -268,7 +268,7 @@ jobs: overwrite: true merge-artifacts: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [compile-warnings-report] steps: - name: Merge Pytest Warnings JSON Artifacts @@ -288,7 +288,7 @@ jobs: # Combine and upload coverage reports. coverage: if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [run-tests] strategy: matrix: From 7c397bf1fc769bac96fdd743c043a0f77a2d528e Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 23 Oct 2024 13:01:42 -0400 Subject: [PATCH 2/6] build: Use a re-usable action to setup mongo. Rather than doing the install ourselves and needing to keep the debion source URL up-to-date, use a community action to handle installing mongo. --- .github/workflows/unit-tests.yml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 03d3ce60e24b..ef79a4c88f8a 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -90,19 +90,10 @@ jobs: activate = 1 EOF - - name: install mongo version - run: | - if [[ "${{ matrix.mongo-version }}" != "4.4" ]]; then - wget -qO - https://www.mongodb.org/static/pgp/server-${{ matrix.mongo-version }}.asc | sudo apt-key add - - echo "deb https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${{ matrix.mongo-version }} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${{ matrix.mongo-version }}.list - sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*" - fi - - - name: start mongod server for tests - run: | - sudo mkdir -p /data/db - sudo chmod -R a+rw /data/db - mongod & + - name: Start MongoDB + uses: supercharge/mongodb-github-action@1.11.0 + with: + mongodb-version: ${{ matrix.mongo-version }} - name: Setup Python uses: actions/setup-python@v5 From d27bd5be0492acdd2c567efc0c6395509ae6bfce Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 23 Oct 2024 13:49:56 -0400 Subject: [PATCH 3/6] chore: Unpin and upgrade xmlsec. --- requirements/constraints.txt | 5 ----- requirements/edx/base.txt | 6 ++---- requirements/edx/development.txt | 3 +-- requirements/edx/doc.txt | 3 +-- requirements/edx/testing.txt | 3 +-- 5 files changed, 5 insertions(+), 15 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 1fe478a4b9ae..5fd6a5fe371a 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -191,8 +191,3 @@ social-auth-app-django<=5.4.1 # # Date: 2024-10-14 # # The edx-enterprise is currently using edx-rest-api-client==5.7.1, which needs to be updated first. # edx-rest-api-client==5.7.1 - -# Date: 2024-04-24 -# xmlsec==1.3.14 breaking tests or all builds, can be removed once a fix is available -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35264 -xmlsec<1.3.14 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index f1a7db4c80ed..668ce09b5c66 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -1285,10 +1285,8 @@ xblock-utils==4.0.0 # via # edx-sga # xblock-poll -xmlsec==1.3.13 - # via - # -c requirements/edx/../constraints.txt - # python3-saml +xmlsec==1.3.14 + # via python3-saml xss-utils==0.6.0 # via -r requirements/edx/kernel.in yarl==1.17.0 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index eec25125f84b..1cc881594c8a 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -2270,9 +2270,8 @@ xblock-utils==4.0.0 # -r requirements/edx/testing.txt # edx-sga # xblock-poll -xmlsec==1.3.13 +xmlsec==1.3.14 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # python3-saml diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 62f435c49719..7f43d85ae768 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -1579,9 +1579,8 @@ xblock-utils==4.0.0 # -r requirements/edx/base.txt # edx-sga # xblock-poll -xmlsec==1.3.13 +xmlsec==1.3.14 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # python3-saml xss-utils==0.6.0 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 58f778af500e..e7507e63d85b 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1678,9 +1678,8 @@ xblock-utils==4.0.0 # -r requirements/edx/base.txt # edx-sga # xblock-poll -xmlsec==1.3.13 +xmlsec==1.3.14 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # python3-saml xss-utils==0.6.0 From 737b8e99dd8d551d566462a508e0aa4210360f9b Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 23 Oct 2024 13:53:31 -0400 Subject: [PATCH 4/6] build: Test on both the old and new ubuntu version. --- .github/workflows/unit-tests.yml | 35 ++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ef79a4c88f8a..e691e16e47f1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -15,7 +15,7 @@ concurrency: jobs: run-tests: name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.os-version }} strategy: matrix: python-version: @@ -43,22 +43,27 @@ jobs: - "xmodule-with-cms" mongo-version: - "7.0" + os-version: + - ubuntu-24.04 - # We only need to test older versions of Mongo with modules that directly interface with Mongo (that is: xmodule.modulestore) - # This code is left here as an example for future refernce in case we need to reduce the number of shards we're - # testing but still have good confidence with older versions of mongo. We use Mongo 4.4 in the example. + # It's useful to run some subset of the tests on the older version of Ubuntu + # so that we don't spend too many resources on this but can find major issues quickly + # while we're in a situation where we support two versions. This section may be commented + # out when not in use to easily add/drop future support for any given major dependency. # - # exclude: - # - mongo-version: "4.4" - # include: - # - shard_name: "xmodule-with-cms" - # python-version: "3.11" - # django-version: "pinned" - # mongo-version: "4.4" - # - shard_name: "xmodule-with-lms" - # python-version: "3.11" - # django-version: "pinned" - # mongo-version: "4.4" + # We're testing the older version of Ubuntu and running the xmodule tests since those rely on many + # dependent complex libraries and will hopefully catch most issues quickly. + include: + - shard_name: "xmodule-with-cms" + python-version: "3.11" + django-version: "pinned" + mongo-version: "7.0" + os-version: "ubuntu-22.04" + - shard_name: "xmodule-with-lms" + python-version: "3.11" + django-version: "pinned" + mongo-version: "7.0" + os-version: "ubuntu-22.04" steps: - name: checkout repo From a1bc4760aae9566e87416661a7c20d22b90bcc9f Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 24 Oct 2024 14:13:08 -0400 Subject: [PATCH 5/6] build: Unpin lxml. libxmlsec and lxml need to be updated in lockstep and the version we had wouldn't work with Ubuntu 24.04 so unpinning lxml along with libxmlsec to see if that resolves the build and dependency issues. --- requirements/constraints.txt | 7 ------- requirements/edx/kernel.in | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 5fd6a5fe371a..9f26a8f73617 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -105,13 +105,6 @@ event-tracking==3.0.0 # https://github.com/openedx/edx-platform/issues/31616 libsass==0.10.0 -# Date: 2024-04-30 -# lxml>=5.0 introduced breaking changes related to system dependencies -# lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean] -# This constraint can be removed once we upgrade to Python 3.11 -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35272 -lxml<5.0 - # Date: 2018-12-14 # markdown>=3.4.0 has failures due to internal refactorings which causes the tests to fail # pinning the version untill the issue gets resolved in the package itself diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index a5b510742ac7..7323c243accf 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -103,7 +103,7 @@ importlib_metadata # Used to access entry_points in i18n_api pl jsonfield # Django model field for validated JSON; used in several apps laboratory # Library for testing that code refactors/infrastructure changes produce identical results importlib_metadata # Used to access entry_points in i18n_api plugin -lxml # XML parser +lxml[html_clean] # XML parser lti-consumer-xblock>=7.3.0 mako # Primary template language used for server-side page rendering Markdown # Convert text markup to HTML; used in capa problems, forums, and course wikis From 5cb29848c3dbfe3072b48b30d56530bed67918b4 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 24 Oct 2024 14:18:22 -0400 Subject: [PATCH 6/6] chore: Update lxml Run `make upgrade-package package='lxml[html_clean]'` to update lxml and then `make compile-requirements` --- requirements/edx-sandbox/base.in | 2 +- requirements/edx-sandbox/base.txt | 6 ++++-- requirements/edx/base.txt | 6 ++++-- requirements/edx/development.txt | 9 +++++++-- requirements/edx/doc.txt | 8 ++++++-- requirements/edx/testing.txt | 8 ++++++-- scripts/user_retirement/requirements/base.txt | 6 ++---- scripts/user_retirement/requirements/testing.txt | 2 +- 8 files changed, 31 insertions(+), 16 deletions(-) diff --git a/requirements/edx-sandbox/base.in b/requirements/edx-sandbox/base.in index 4c1a61bc2723..0c331ce95d62 100644 --- a/requirements/edx-sandbox/base.in +++ b/requirements/edx-sandbox/base.in @@ -2,7 +2,7 @@ chem # A helper library for chemistry calculations cryptography # Implementations of assorted cryptography algorithms -lxml # XML parser +lxml[html_clean] # XML parser matplotlib # 2D plotting library networkx # Utilities for creating, manipulating, and studying network graphs nltk # Natural language processing; used by the chem package diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index eb74898596ff..8cf0cbb82904 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -26,11 +26,13 @@ joblib==1.4.2 # via nltk kiwisolver==1.4.7 # via matplotlib -lxml==4.9.4 +lxml[html-clean,html_clean]==5.3.0 # via - # -c requirements/edx-sandbox/../constraints.txt # -r requirements/edx-sandbox/base.in + # lxml-html-clean # openedx-calc +lxml-html-clean==0.3.1 + # via lxml markupsafe==3.0.2 # via # chem diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 668ce09b5c66..86509c651187 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -708,19 +708,21 @@ loremipsum==1.0.5 # via ora2 lti-consumer-xblock==9.11.3 # via -r requirements/edx/kernel.in -lxml==4.9.4 +lxml[html-clean,html_clean]==5.3.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in # edx-i18n-tools # edxval # lti-consumer-xblock + # lxml-html-clean # olxcleaner # openedx-calc # ora2 # python3-saml # xblock # xmlsec +lxml-html-clean==0.3.1 + # via lxml mailsnake==1.6.4 # via -r requirements/edx/bundled.in mako==1.3.6 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 1cc881594c8a..97737e8d2916 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1183,14 +1183,14 @@ lti-consumer-xblock==9.11.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -lxml==4.9.4 +lxml[html-clean]==5.3.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-i18n-tools # edxval # lti-consumer-xblock + # lxml-html-clean # olxcleaner # openedx-calc # ora2 @@ -1198,6 +1198,11 @@ lxml==4.9.4 # python3-saml # xblock # xmlsec +lxml-html-clean==0.3.1 + # via + # -r requirements/edx/doc.txt + # -r requirements/edx/testing.txt + # lxml mailsnake==1.6.4 # via # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 7f43d85ae768..a45b434f1f13 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -850,19 +850,23 @@ loremipsum==1.0.5 # ora2 lti-consumer-xblock==9.11.3 # via -r requirements/edx/base.txt -lxml==4.9.4 +lxml[html-clean]==5.3.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # edx-i18n-tools # edxval # lti-consumer-xblock + # lxml-html-clean # olxcleaner # openedx-calc # ora2 # python3-saml # xblock # xmlsec +lxml-html-clean==0.3.1 + # via + # -r requirements/edx/base.txt + # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt mako==1.3.6 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e7507e63d85b..e6a13cb42c55 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -899,13 +899,13 @@ loremipsum==1.0.5 # ora2 lti-consumer-xblock==9.11.3 # via -r requirements/edx/base.txt -lxml==4.9.4 +lxml[html-clean]==5.3.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # edx-i18n-tools # edxval # lti-consumer-xblock + # lxml-html-clean # olxcleaner # openedx-calc # ora2 @@ -913,6 +913,10 @@ lxml==4.9.4 # python3-saml # xblock # xmlsec +lxml-html-clean==0.3.1 + # via + # -r requirements/edx/base.txt + # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt mako==1.3.6 diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 288d660b14be..704baaff2c79 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -77,10 +77,8 @@ jmespath==1.0.1 # via # boto3 # botocore -lxml==4.9.4 - # via - # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt - # zeep +lxml==5.3.0 + # via zeep more-itertools==10.5.0 # via simple-salesforce newrelic==10.2.0 diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index da4ce1b0391a..4cb3de607db7 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -116,7 +116,7 @@ jmespath==1.0.1 # -r scripts/user_retirement/requirements/base.txt # boto3 # botocore -lxml==4.9.4 +lxml==5.3.0 # via # -r scripts/user_retirement/requirements/base.txt # zeep