feat!: upgrade to Django 4.2 LTS (#349)

build: update CI

Co-authored-by: Muhammad Soban Javed <[email protected]>

.github/workflows/ci.yml

@@ -14,22 +14,18 @@ jobs:
       max-parallel: 4
       matrix:
         python-version: ['py38']
-        django-version: ['django32', 'django42']
-        db-version: ['mysql57', 'mysql80']
-        # excluding mysql5.7 with Django 4.2 since Django 4.2 has 
-        # dropped support for MySQL<8
-        exclude:
-          - django-version: 'django42'
-            db-version: 'mysql57'
+        django-version: ['django42']
+        db-version: ['mysql80']
 
     steps:
     - uses: actions/checkout@v2
     - name: Start container
       run: |
         docker-compose -f .travis/docker-compose-travis.yml up -d
-    - name: Install Dependencies
+    - name: Install system packages
       run: |
-        pip install -r requirements/travis.txt
+        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \
+        /bin/bash -c "apt-get update && apt-get install python3-dev default-libmysqlclient-dev build-essential pkg-config"  
     - name: Run Tests
       run: |
         docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.travis/run_tests.sh

.travis/docker-compose-travis.yml

@@ -1,16 +1,6 @@
 version: "2"
 
 services:
-  mysql57:
-    image: mysql:5.7
-    container_name: mysql57
-    command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci
-    environment:
-      MYSQL_ROOT_PASSWORD: ""
-      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
-      MYSQL_USER: "notes001"
-      MYSQL_PASSWORD: "secret"
-      MYSQL_DATABASE: "edx_notes_api"
   mysql80:
     image: mysql:8.0
     container_name: mysql80
@@ -49,7 +39,6 @@ services:
       - ..:/edx/app/edx_notes_api/edx_notes_api
     command: tail -f /dev/null
     depends_on:
-      - "mysql57"
       - "mysql80"
       - "es"
     environment:

Makefile

@@ -65,8 +65,21 @@ develop: requirements test.requirements
 piptools: ## install pinned version of pip-compile and pip-sync
 	pip install -r requirements/pip-tools.txt
 
+define COMMON_CONSTRAINTS_TEMP_COMMENT
+# This is a temporary solution to override the real common_constraints.txt\n# In edx-lint, until the pyjwt constraint in edx-lint has been removed.\n# See BOM-2721 for more details.\n# Below is the copied and edited version of common_constraints\n
+endef
+COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
+.PHONY: $(COMMON_CONSTRAINTS_TXT)
+$(COMMON_CONSTRAINTS_TXT):
+	wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)"
+	echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)
+
 upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: piptools ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+upgrade: piptools $(COMMON_CONSTRAINTS_TXT)  ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+	# This is a temporary solution to override the real common_constraints.txt
+	# In edx-lint, until the pyjwt constraint in edx-lint has been removed.
+	# See BOM-271 for more details.
+	sed -i.'' 's/Django<4.0//g' requirements/common_constraints.txt
 	# Make sure to compile files after any other files they include!
 	pip-compile --upgrade -o requirements/travis.txt requirements/travis.in
 	pip-compile --upgrade --rebuild --allow-unsafe -o requirements/pip.txt requirements/pip.in
@@ -77,4 +90,4 @@ upgrade: piptools ## update the requirements/*.txt files with the latest package
 	pip-compile --upgrade -o requirements/test.txt requirements/test.in
 	# Let tox control the Django version for tests
 	grep -e "^django==" requirements/base.txt > requirements/django.txt
-	sed -i '/^[dD]jango==/d' requirements/test.txt
+	sed -i.'' '/^[dD]jango==/d' requirements/test.txt

requirements/base.txt

@@ -4,46 +4,43 @@
 #
 #    make upgrade
 #
-asgiref==3.6.0
+asgiref==3.7.2
     # via django
-certifi==2022.12.7
+backports-zoneinfo==0.2.1
+    # via django
+certifi==2023.7.22
     # via
     #   elasticsearch
     #   requests
-cffi==1.15.1
+cffi==1.16.0
     # via
     #   cryptography
     #   pynacl
 chardet==3.0.4
     # via requests
-click==8.1.3
+click==8.1.7
     # via edx-django-utils
-coreapi==2.3.3
-    # via drf-yasg
-coreschema==0.0.4
-    # via
-    #   coreapi
-    #   drf-yasg
-cryptography==40.0.1
+cryptography==41.0.4
     # via pyjwt
-django==3.2.20
+django==4.2.6
     # via
-    #   -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
+    #   -c requirements/constraints.txt
     #   -r requirements/base.in
     #   django-cors-headers
     #   django-crum
     #   django-nine
+    #   django-waffle
     #   djangorestframework
     #   drf-jwt
     #   drf-yasg
     #   edx-django-release-util
     #   edx-django-utils
     #   edx-drf-extensions
-django-cors-headers==3.14.0
+django-cors-headers==4.2.0
     # via -r requirements/base.in
 django-crum==0.7.9
     # via edx-django-utils
-django-elasticsearch-dsl==7.3
+django-elasticsearch-dsl==7.4
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
@@ -52,7 +49,7 @@ django-elasticsearch-dsl-drf==0.22.5
     # via -r requirements/base.in
 django-nine==0.2.7
     # via django-elasticsearch-dsl-drf
-django-waffle==3.0.0
+django-waffle==4.0.0
     # via
     #   edx-django-utils
     #   edx-drf-extensions
@@ -65,21 +62,21 @@ djangorestframework==3.14.0
     #   edx-drf-extensions
 drf-jwt==1.19.2
     # via edx-drf-extensions
-drf-yasg==1.21.5
+drf-yasg==1.21.7
     # via -r requirements/base.in
-edx-django-release-util==1.2.0
+edx-django-release-util==1.3.0
     # via -r requirements/base.in
-edx-django-utils==5.3.0
+edx-django-utils==5.7.0
     # via
     #   -r requirements/base.in
     #   edx-drf-extensions
-edx-drf-extensions==8.4.1
+edx-drf-extensions==8.10.0
     # via -r requirements/base.in
-edx-opaque-keys==2.3.0
+edx-opaque-keys==2.5.1
     # via edx-drf-extensions
 elasticsearch==7.13.4
     # via
-    #   -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
+    #   -c requirements/common_constraints.txt
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
     #   django-elasticsearch-dsl-drf
@@ -90,46 +87,35 @@ elasticsearch-dsl==7.4.1
     #   -r requirements/base.in
     #   django-elasticsearch-dsl
     #   django-elasticsearch-dsl-drf
-future==0.18.3
-    # via pyjwkest
-gunicorn==20.1.0
+gunicorn==21.2.0
     # via -r requirements/base.in
 idna==2.10
     # via requests
 inflection==0.5.1
     # via drf-yasg
-itypes==1.2.0
-    # via coreapi
-jinja2==3.1.2
-    # via coreschema
-markupsafe==2.1.2
-    # via jinja2
-mysqlclient==2.1.1
+mysqlclient==2.2.0
     # via -r requirements/base.in
 newrelic==4.8.0.110
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
     #   edx-django-utils
-packaging==23.0
+packaging==23.2
     # via
     #   django-nine
     #   drf-yasg
+    #   gunicorn
 path-py==9.1
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
 pbr==5.11.1
     # via stevedore
-psutil==5.9.4
+psutil==5.9.5
     # via edx-django-utils
 pycparser==2.21
     # via cffi
-pycryptodomex==3.17
-    # via pyjwkest
-pyjwkest==1.4.2
-    # via edx-drf-extensions
-pyjwt[crypto]==2.6.0
+pyjwt[crypto]==2.8.0
     # via
     #   -r requirements/base.in
     #   drf-jwt
@@ -142,51 +128,42 @@ python-dateutil==2.4.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
-    #   edx-drf-extensions
     #   elasticsearch-dsl
-pytz==2023.3
+pytz==2023.3.post1
     # via
-    #   django
     #   djangorestframework
     #   drf-yasg
-pyyaml==6.0
-    # via edx-django-release-util
+pyyaml==6.0.1
+    # via
+    #   drf-yasg
+    #   edx-django-release-util
 requests==2.25.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
-    #   coreapi
     #   edx-drf-extensions
-    #   pyjwkest
-ruamel-yaml==0.17.21
-    # via drf-yasg
-ruamel-yaml-clib==0.2.7
-    # via ruamel-yaml
 semantic-version==2.10.0
     # via edx-drf-extensions
 six==1.16.0
     # via
     #   django-elasticsearch-dsl
     #   django-elasticsearch-dsl-drf
     #   edx-django-release-util
-    #   edx-drf-extensions
     #   elasticsearch-dsl
-    #   pyjwkest
     #   python-dateutil
-sqlparse==0.4.3
+sqlparse==0.4.4
     # via django
-stevedore==5.0.0
+stevedore==5.1.0
     # via
     #   edx-django-utils
     #   edx-opaque-keys
-uritemplate==4.1.1
+typing-extensions==4.8.0
     # via
-    #   coreapi
-    #   drf-yasg
-urllib3==1.26.15
+    #   asgiref
+    #   edx-opaque-keys
+uritemplate==4.1.1
+    # via drf-yasg
+urllib3==1.26.17
     # via
     #   elasticsearch
     #   requests
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

requirements/common_constraints.txt

@@ -0,0 +1,32 @@
+# This is a temporary solution to override the real common_constraints.txt
+# In edx-lint, until the pyjwt constraint in edx-lint has been removed.
+# See BOM-2721 for more details.
+# Below is the copied and edited version of common_constraints
+
+# A central location for most common version constraints
+# (across edx repos) for pip-installation.
+#
+# Similar to other constraint files this file doesn't install any packages.
+# It specifies version constraints that will be applied if a package is needed.
+# When pinning something here, please provide an explanation of why it is a good
+# idea to pin this package across all edx repos, Ideally, link to other information
+# that will help people in the future to remove the pin when possible.
+# Writing an issue against the offending project and linking to it here is good.
+#
+# Note: Changes to this file will automatically be used by other repos, referencing
+#  this file from Github directly. It does not require packaging in edx-lint.
+
+
+# using LTS django version
+
+
+# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process.
+# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html
+elasticsearch<7.14.0
+
+# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected
+django-simple-history==3.0.0
+
+# tox>4.0.0 isn't yet compatible with many tox plugins, causing CI failures in almost all repos.
+# Details can be found in this discussion: https://github.com/tox-dev/tox/discussions/1810
+tox<4.0.0

requirements/constraints.txt

@@ -1,4 +1,4 @@
-# Version constraints for pip installation.
+# Version constraints for pip-installation.
 #
 # This file doesn't install any packages. It specifies version constraints
 # that will be applied if a package is needed.
@@ -8,8 +8,11 @@
 # pin when possible.  Writing an issue against the offending project and
 # linking to it here is good.
 
-# This file contains all common constraints for edx-repos
--c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
+-c common_constraints.txt
+
+# Keeping Django to latest Django 4.2 LTS
+# This constraint can be removed once the global constraint has been updated
+Django<4.3
 
 astroid==1.6.6
 requests==2.25.0

requirements/django.txt

@@ -1 +1 @@
-django==3.2.20
+django==4.2.6

requirements/pip-tools.txt

@@ -4,20 +4,27 @@
 #
 #    make upgrade
 #
-build==0.10.0
+build==1.0.3
     # via pip-tools
-click==8.1.3
+click==8.1.7
     # via pip-tools
-packaging==23.0
+importlib-metadata==6.8.0
     # via build
-pip-tools==6.12.3
+packaging==23.2
+    # via build
+pip-tools==7.3.0
     # via -r requirements/pip-tools.in
 pyproject-hooks==1.0.0
     # via build
 tomli==2.0.1
-    # via build
-wheel==0.40.0
+    # via
+    #   build
+    #   pip-tools
+    #   pyproject-hooks
+wheel==0.41.2
     # via pip-tools
+zipp==3.17.0
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip