From dafcd0c38aff96ae8f43ad26eaf565879ea42698 Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 8 Dec 2022 12:44:13 -0500 Subject: [PATCH] Bump dependencies and speedup tests --- .github/dependabot.yml | 69 ++++++++++++++++++- .github/disabled-dependabot.yml | 57 --------------- .github/workflows/codeql-analysis.yml | 58 ++++++++-------- .../lint-and-build-global-scoreboard.yml | 7 +- .../lint-and-build-tournament-scheduler.yml | 7 +- .github/workflows/lint-python.yml | 45 ++++++------ backend/api/global_scoreboard_api.py | 2 +- backend/models/core_models.py | 16 +++-- backend/models/global_scoreboard_models.py | 2 +- pyproject.toml | 2 + scripts/{Deploy_react_apps.sh => Deploy.sh} | 1 + scripts/install.ps1 | 2 +- scripts/requirements-dev.txt | 27 ++++++++ scripts/requirements.txt | 47 ++++--------- 14 files changed, 183 insertions(+), 159 deletions(-) delete mode 100644 .github/disabled-dependabot.yml rename scripts/{Deploy_react_apps.sh => Deploy.sh} (88%) create mode 100644 scripts/requirements-dev.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 34f00a2b..7eb8db8a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,69 @@ +# Reconsider Dependabot if they ever add grouped updates: https://github.com/dependabot/dependabot-core/issues/1190 + +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + version: 2 updates: - package-ecosystem: "npm" - directory: "/**" - allow: - - dependency-type: "production" + directory: "/tournament-scheduler" + target-branch: "develop" + schedule: + interval: "weekly" + labels: + - "dependencies" + - "tournament-scheduler" + assignees: + - "Avasam" + reviewers: + - "Avasam" + ignore: + - dependency-name: "*" + # Temporarily adding version-update:semver-minor to the ignore as dependabot + # still wants to increase the minor version in the lockfile only despite + # versioning-strategy: increase-if-necessary. This is a know issue: + # https://github.com/dependabot/dependabot-core/issues/3891 + update-types: + ["version-update:semver-patch", "version-update:semver-minor"] + versioning-strategy: increase-if-necessary + + - package-ecosystem: "npm" + directory: "/global-scoreboard" + target-branch: "develop" + schedule: + interval: "weekly" + labels: + - "dependencies" + assignees: + - "Avasam" + reviewers: + - "Avasam" + ignore: + - dependency-name: "*" + # Temporarily adding version-update:semver-minor to the ignore as dependabot + # still wants to increase the minor version in the lockfile only despite + # versioning-strategy: increase-if-necessary. This is a know issue: + # https://github.com/dependabot/dependabot-core/issues/3891 + update-types: + ["version-update:semver-patch", "version-update:semver-minor"] + versioning-strategy: increase-if-necessary + + - package-ecosystem: "pip" + directory: "/scripts" + target-branch: "develop" + schedule: + interval: "weekly" + labels: + - "dependencies" + assignees: + - "Avasam" + reviewers: + - "Avasam" + + # Check for updates to GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/disabled-dependabot.yml b/.github/disabled-dependabot.yml deleted file mode 100644 index e5808d5f..00000000 --- a/.github/disabled-dependabot.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Reconsider Dependabot if they ever add grouped updates: https://github.com/dependabot/dependabot-core/issues/1190 - -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "npm" - directory: "/tournament-scheduler" - target-branch: "develop" - open-pull-requests-limit: 10 - schedule: - interval: "weekly" - labels: - - "dependencies" - - "tournament-scheduler" - assignees: - - "Avasam" - reviewers: - - "Avasam" - ignore: - - dependency-name: "*" - # Temporarily adding version-update:semver-minor to the ignore as dependabot - # still wants to increase the minor version in the lockfile only despite - # versioning-strategy: increase-if-necessary. This is a know issue: - # https://github.com/dependabot/dependabot-core/issues/3891 - update-types: ["version-update:semver-patch", "version-update:semver-minor"] - versioning-strategy: increase-if-necessary - - - package-ecosystem: "npm" - directory: "/global-scoreboard" - target-branch: "develop" - open-pull-requests-limit: 10 - schedule: - interval: "weekly" - labels: - - "dependencies" - assignees: - - "Avasam" - reviewers: - - "Avasam" - ignore: - - dependency-name: "*" - # Temporarily adding version-update:semver-minor to the ignore as dependabot - # still wants to increase the minor version in the lockfile only despite - # versioning-strategy: increase-if-necessary. This is a know issue: - # https://github.com/dependabot/dependabot-core/issues/3891 - update-types: ["version-update:semver-patch", "version-update:semver-minor"] - versioning-strategy: increase-if-necessary - - # Check for updates to GitHub Actions - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bd56b631..b07360b2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,12 +13,12 @@ name: "CodeQL" on: push: - branches: [ main, develop ] + branches: [main, develop] pull_request: # The branches below must be a subset of the branches above - branches: [ develop ] + branches: [develop] schedule: - - cron: '26 13 * * 6' + - cron: "26 13 * * 6" jobs: analyze: @@ -28,40 +28,40 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'javascript', 'python' ] + language: ["javascript", "python"] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] # Learn more: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v3 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language - #- run: | - # make bootstrap - # make release + #- run: | + # make bootstrap + # make release - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/lint-and-build-global-scoreboard.yml b/.github/workflows/lint-and-build-global-scoreboard.yml index 3e5fb3bc..7795cc5e 100644 --- a/.github/workflows/lint-and-build-global-scoreboard.yml +++ b/.github/workflows/lint-and-build-global-scoreboard.yml @@ -4,14 +4,17 @@ on: push: branches: - main + - develop paths: - - 'global-scoreboard/**' + - "global-scoreboard/**" + - ".github/workflows/lint-and-build-global-scoreboard.yml" pull_request: branches: - main - develop paths: - - 'global-scoreboard/**' + - "global-scoreboard/**" + - ".github/workflows/lint-and-build-global-scoreboard.yml" jobs: ESLint: runs-on: ubuntu-latest diff --git a/.github/workflows/lint-and-build-tournament-scheduler.yml b/.github/workflows/lint-and-build-tournament-scheduler.yml index b44c2a56..0135a886 100644 --- a/.github/workflows/lint-and-build-tournament-scheduler.yml +++ b/.github/workflows/lint-and-build-tournament-scheduler.yml @@ -4,14 +4,17 @@ on: push: branches: - main + - develop paths: - - 'tournament-scheduler/**' + - "tournament-scheduler/**" + - ".github/workflows/lint-and-build-tournament-scheduler.yml" pull_request: branches: - main - develop paths: - - 'tournament-scheduler/**' + - "tournament-scheduler/**" + - ".github/workflows/lint-and-build-tournament-scheduler.yml" jobs: ESLint: runs-on: ubuntu-latest diff --git a/.github/workflows/lint-python.yml b/.github/workflows/lint-python.yml index 0dacd4c5..0f7e701b 100644 --- a/.github/workflows/lint-python.yml +++ b/.github/workflows/lint-python.yml @@ -4,9 +4,11 @@ on: push: branches: - main + - develop paths: - "**.py" - "**.pyi" + - ".github/workflows/lint-python.yml" pull_request: branches: - main @@ -14,13 +16,18 @@ on: paths: - "**.py" - "**.pyi" + - ".github/workflows/lint-python.yml" env: + # https://help.pythonanywhere.com/pages/ChangingSystemImage/#available-python-versions-for-system-images python-version: "3.10" + # https://help.pythonanywhere.com/pages/ChangingSystemImage/#base-ubuntu-version-for-each-system-image + pythonanywhere-image: "ubuntu-20.04" jobs: + # Jobs that don't need dependencies installed isort: - runs-on: ubuntu-latest + runs-on: ${{ env.pythonanywhere-image }} steps: - name: Checkout ${{ github.repository }}/${{ github.ref }} uses: actions/checkout@v3 @@ -29,15 +36,12 @@ jobs: with: python-version: ${{ env.python-version }} cache: "pip" - cache-dependency-path: "scripts/requirements*.txt" - - name: Install dependencies - run: | - pip install wheel - pip install -r "scripts/requirements.txt" + cache-dependency-path: "scripts/requirements-dev.txt" + - run: pip install isort - name: Analysing the code with ${{ job.name }} run: isort backend/ typings/ --check-only add-trailing-comma: - runs-on: ubuntu-latest + runs-on: ${{ env.pythonanywhere-image }} steps: - name: Checkout ${{ github.repository }}/${{ github.ref }} uses: actions/checkout@v3 @@ -46,12 +50,12 @@ jobs: with: python-version: ${{ env.python-version }} cache: "pip" - cache-dependency-path: "scripts/requirements*.txt" + cache-dependency-path: "scripts/requirements-dev.txt" - run: pip install add-trailing-comma - name: Analysing the code with ${{ job.name }} run: add-trailing-comma $(git ls-files '**.py*') --py36-plus Bandit: - runs-on: ubuntu-latest + runs-on: ${{ env.pythonanywhere-image }} steps: - name: Checkout ${{ github.repository }}/${{ github.ref }} uses: actions/checkout@v3 @@ -60,13 +64,14 @@ jobs: with: python-version: ${{ env.python-version }} cache: "pip" - cache-dependency-path: "scripts/requirements*.txt" + cache-dependency-path: "scripts/requirements-dev.txt" - run: pip install bandit - - run: mv backend/configs.template.py backend/configs.py - name: Analysing the code with ${{ job.name }} run: bandit -n 1 --severity-level medium --recursive backend + + # Jobs that do need dependencies installed Pyright: - runs-on: ubuntu-latest + runs-on: ${{ env.pythonanywhere-image }} steps: - name: Checkout ${{ github.repository }}/${{ github.ref }} uses: actions/checkout@v3 @@ -79,7 +84,7 @@ jobs: - name: Install dependencies run: | pip install wheel - pip install -r "scripts/requirements.txt" + pip install -r "scripts/requirements-dev.txt" - run: mv backend/configs.template.py backend/configs.py - name: Analysing the code with ${{ job.name }} uses: jakebailey/pyright-action@v1 @@ -87,7 +92,7 @@ jobs: working-directory: backend/ extra-args: --warnings Pylint: - runs-on: ubuntu-latest + runs-on: ${{ env.pythonanywhere-image }} steps: - name: Checkout ${{ github.repository }}/${{ github.ref }} uses: actions/checkout@v3 @@ -100,12 +105,12 @@ jobs: - name: Install dependencies run: | pip install wheel - pip install -r "scripts/requirements.txt" + pip install -r "scripts/requirements-dev.txt" - run: mv backend/configs.template.py backend/configs.py - name: Analysing the code with ${{ job.name }} run: pylint backend/ --reports=y --output-format=colorized Flake8: - runs-on: ubuntu-latest + runs-on: ${{ env.pythonanywhere-image }} steps: - name: Checkout ${{ github.repository }}/${{ github.ref }} uses: actions/checkout@v3 @@ -114,11 +119,7 @@ jobs: with: python-version: ${{ env.python-version }} cache: "pip" - cache-dependency-path: "scripts/requirements*.txt" - - name: Install dependencies - run: | - pip install wheel - pip install -r "scripts/requirements.txt" - - run: mv backend/configs.template.py backend/configs.py + cache-dependency-path: "scripts/requirements-dev.txt" + - run: pip install -r "scripts/requirements-dev.txt" - name: Analysing the code with ${{ job.name }} run: flake8 backend/ typings/ diff --git a/backend/api/global_scoreboard_api.py b/backend/api/global_scoreboard_api.py index d97a29ed..f91805b4 100644 --- a/backend/api/global_scoreboard_api.py +++ b/backend/api/global_scoreboard_api.py @@ -45,7 +45,7 @@ def update_player(name_or_id: str): if configs.bypass_update_restrictions: return __do_update_player_bypass_restrictions(name_or_id) # pylint: disable=no-value-for-parameter # TODO: Raise this issue upstream - return __do_update_player(name_or_id) # type: ignore # TODO: Raise this issue upstream + return __do_update_player(name_or_id) # pyright: ignore[reportGeneralTypeIssues] except UserUpdaterError as exception: error_message = f"Error: {exception.args[0]['error']}\n{exception.args[0]['details']}" return error_message, 424 diff --git a/backend/models/core_models.py b/backend/models/core_models.py index 36887370..53a8f68b 100644 --- a/backend/models/core_models.py +++ b/backend/models/core_models.py @@ -312,15 +312,19 @@ def update_schedule( for existing_time_slot in schedule_to_update.time_slots: if time_slot_to_edit["id"] == existing_time_slot.time_slot_id: new_time_slot = existing_time_slot + new_time_slot.date_time = datetime.strptime(time_slot_to_edit["dateTime"], DATETIME_FORMAT) + new_time_slot.maximum_entries = time_slot_to_edit["maximumEntries"] + new_time_slot.participants_per_entry = time_slot_to_edit["participantsPerEntry"] + new_time_slot.schedule_id = schedule_id break # ... otherwise, create a brand new TimeSlot else: - new_time_slot = TimeSlot() # type: ignore - # Do the necessary modifications - new_time_slot.schedule_id = schedule_id - new_time_slot.date_time = datetime.strptime(time_slot_to_edit["dateTime"], DATETIME_FORMAT) - new_time_slot.maximum_entries = time_slot_to_edit["maximumEntries"] - new_time_slot.participants_per_entry = time_slot_to_edit["participantsPerEntry"] + new_time_slot = TimeSlot( + date_time=datetime.strptime(time_slot_to_edit["dateTime"], DATETIME_FORMAT), + maximum_entries=time_slot_to_edit["maximumEntries"], + participants_per_entry=time_slot_to_edit["participantsPerEntry"], + schedule_id=schedule_id, + ) new_time_slots.append(new_time_slot) diff --git a/backend/models/global_scoreboard_models.py b/backend/models/global_scoreboard_models.py index 1c238a46..51ab4a2e 100644 --- a/backend/models/global_scoreboard_models.py +++ b/backend/models/global_scoreboard_models.py @@ -127,7 +127,7 @@ def get_points_distribution_dto(self) -> PointsDistributionDto: ] def fetch_and_set_user_code_and_name(self) -> None: - url = "https://www.speedrun.com/api/v1/users/{user}".format(user=self._id) + url = "https://www.speedrun.com/api/v1/users/{user}".format(user=self._id) # pylint: disable=C0209 infos: SrcProfileDto = get_file(url, {}, "http_cache")["data"] self._id = infos["id"] diff --git a/pyproject.toml b/pyproject.toml index 005c7510..1e65aded 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,8 @@ reportMissingSuperCall="none" # False positives on base classes reportPropertyTypeMismatch="error" reportUninitializedInstanceVariable="error" reportUnnecessaryTypeIgnoreComment="error" +# Use `pyright: ignore`, not `type: ignore` +enableTypeIgnoreComments=false # Ignore must be specified for Pylance to stop displaying errors ignore = [ # We expect stub files to be incomplete or contain useless statements diff --git a/scripts/Deploy_react_apps.sh b/scripts/Deploy.sh similarity index 88% rename from scripts/Deploy_react_apps.sh rename to scripts/Deploy.sh index 698dd09d..4fed68cc 100644 --- a/scripts/Deploy_react_apps.sh +++ b/scripts/Deploy.sh @@ -6,3 +6,4 @@ if [ -f global-scoreboard-build.zip ]; then rm -rfv global-scoreboard/build unzip -o global-scoreboard-build.zip -d global-scoreboard/ fi +pip install -r scripts/requirements.txt diff --git a/scripts/install.ps1 b/scripts/install.ps1 index c39ef4f6..bfdcb012 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -7,5 +7,5 @@ If ($IsWindows) { # Ensures installation tools are up to date. This also aliases pip to pip3 on MacOS. python3 -m pip install wheel pip setuptools --upgrade -pip install -r "$PSScriptRoot/requirements.txt" --upgrade +pip install -r "$PSScriptRoot/requirements-dev.txt" --upgrade npm i --global pyright@latest diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt new file mode 100644 index 00000000..231c8760 --- /dev/null +++ b/scripts/requirements-dev.txt @@ -0,0 +1,27 @@ +-r requirements.txt + +# Linters +bandit +flake8>=5,<6 # flake8-pyi deprecation warnings # flake8-quotes doesn't support v6 yet +flake8-builtins +flake8-bugbear +flake8-class-attributes-order +flake8-comprehensions>=3.8 # flake8 5 support +flake8-datetimez +flake8-pyi>=22.11.0 # flake8 6 support +flake8-quotes +flake8-simplify +pep8-naming +pylint>=2.14,<3.0.0 # New checks # 3.0 still in pre-release +pylint-flask +pylint-flask-sqlalchemy +# Formatters +add-trailing-comma>=2.3.0 # Added support for with statement +autopep8>=2.0.0 # New checks +isort +unify + +# types +types-Flask-SQLAlchemy +types-httplib2 +types-requests diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 1c1de14b..8ce3a99d 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,37 +1,14 @@ -flask -flask_sqlalchemy -httplib2 -mysql-connector -pyjwt +# Pythonanywhere pre-installed +# https://www.pythonanywhere.com/batteries_included/ +# Set image to "haggis" and Python version to "3.10" +Flask>=2.1.2 +Flask-SQLAlchemy>=2.5.1 +httplib2>=0.20.4 +mysql-connector-python>=8.0.29 +PyJWT>=2.4.0 +requests>=2.28.1 +requests-cache>=0.9.4 +SQLAlchemy>=1.4.36 +# Custom requirements ratelimiter redislite ; sys_platform != 'win32' -requests -requests-cache>=0.9.3 -sqlalchemy - -# Linters -bandit -flake8>=5,<6 # flake8-pyi deprecation warnings # flake8-quotes doesn't support v6 yet -flake8-builtins -flake8-bugbear -flake8-class-attributes-order -flake8-comprehensions>=3.8 # flake8 5 support -flake8-datetimez -flake8-pyi>=22.11.0 # flake8 6 support -flake8-quotes -flake8-simplify -pep8-naming -pylint>=2.14,<3.0.0 # New checks # 3.0 still in pre-release -pylint-flask -pylint-flask-sqlalchemy -# Formatters -add-trailing-comma>=2.3.0 # Added support for with statement -autopep8>=2.0.0 # New checks -isort -unify - -# types -types-Flask # TODO: Deprecated, update Flask instead! -types-Flask-SQLAlchemy -types-httplib2 -types-requests