From 1dc4a4fc7d65a7a0d64bbfd111e141a4f9a5960a Mon Sep 17 00:00:00 2001 From: Kunal Mehta Date: Thu, 15 Feb 2024 15:52:53 -0500 Subject: [PATCH 1/2] Standardize `make lint` target in components In some places the `lint` target is just flake8 and in others it runs all linters. Adopt the latter definition to ensure that `make lint` runs all tools that don't run any code. --- client/Makefile | 9 ++++++--- export/Makefile | 10 +++++++--- log/Makefile | 5 ++++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/client/Makefile b/client/Makefile index 92d0485ac..afa117fc8 100644 --- a/client/Makefile +++ b/client/Makefile @@ -109,12 +109,15 @@ test-functional: ## Run the functional tests test-sdk: ## Run just the sdk tests @poetry run pytest -v $(STESTS) -.PHONY: lint -lint: ## Run the linters +.PHONY: flake8 +flake8: ## Run flake8 linting @poetry run flake8 securedrop_client tests +.PHONY: lint +lint: check-black check-isort flake8 mypy semgrep ## Run all linters + .PHONY: check -check: clean check-black check-isort semgrep lint mypy test-random test-integration test-functional ## Run the full CI test suite +check: clean lint test-random test-integration test-functional ## Run the full CI test suite # Explanation of the below shell command should it ever break. # 1. Set the field separator to ": ##" and any make targets that might appear between : and ## diff --git a/export/Makefile b/export/Makefile index ad65eaade..ba88300a9 100644 --- a/export/Makefile +++ b/export/Makefile @@ -1,8 +1,12 @@ .PHONY: all all: help +.PHONY: lint +lint: check-black flake8 mypy semgrep ## Run all linters + + .PHONY: check -check: lint mypy semgrep test check-black ## Run linter and tests +check: lint test ## Run linter and tests .PHONY: check-black check-black: ## Check Python source code formatting with black @@ -13,8 +17,8 @@ TESTS ?= tests test: ## Run tests poetry run pytest -v --cov-report html --cov-report term-missing --cov=securedrop_export $$TESTS -.PHONY: lint -lint: ## Run linter +.PHONY: flake8 +flake8: ## Run flake8 linter poetry run flake8 securedrop_export/ tests/ .PHONY: mypy diff --git a/log/Makefile b/log/Makefile index 0d5febc56..79180d4cf 100644 --- a/log/Makefile +++ b/log/Makefile @@ -2,7 +2,10 @@ DEFAULT_GOAL: help SHELL := /bin/bash .PHONY: check -check: flake8 mypy test +check: lint test + +.PHONY: lint +lint: flake8 mypy ## Run all linters .PHONY: flake8 flake8: ## Run flake8 to lint Python files From 3bc1e74e51ece3cce748cab165935815bedd2b27 Mon Sep 17 00:00:00 2001 From: Kunal Mehta Date: Thu, 15 Feb 2024 18:47:37 -0500 Subject: [PATCH 2/2] Move remaining CircleCI jobs to GitHub Actions ci.yml's "component-lint" job runs `make lint` in all four components, and varies by Debian version (except proxy/bookworm). test.yml's "component" job runs `make test` in all components except client. The "client" job runs the test-functional, test-integration, and test-random targets and installs the extra dependencies needed (now in client/Makefile). test.yml also has "internationalization", which checks strings have been extracted and are reproducible. Some parts of GitHub's UI makes it inconvenient to have a ton of jobs in one file, so tests have been split out into their own for solely that reason. Dependencies that aren't needed were dropped as part of this (e.g. libnotify-bin, libmagic1, python3-dev, etc.). Same with the `export PYTHONPATH=$PYTHONPATH:.` pattern (the current directory is always on the Python path...). --- .circleci/config.yml | 410 ------------------------------------- .github/workflows/ci.yml | 46 +++++ .github/workflows/test.yml | 124 +++++++++++ client/Makefile | 9 + 4 files changed, 179 insertions(+), 410 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/test.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 26e139e0e..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,410 +0,0 @@ ---- -common-steps: - - &client_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &client_install_testing_dependencies - run: - name: Install testing dependencies - command: | - set -e - apt update && apt install -y git gnupg libarchive13 libmagic1 libqt5x11extras5 make python3-tk python3-dev gnupg python3-venv sqlite3 xvfb - cd client - poetry install --no-ansi - - - &client_configure_locales - run: - name: Configure locales - command: | - set -e - apt update && apt install -y locales - echo "en_US ISO-8859-1" >> /etc/locale.gen - echo "en_US UTF-8" >> /etc/locale.gen - locale-gen - - - &client_run_unit_tests - run: - name: Install requirements and run unit tests - command: | - set -e - cd client - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test-random - - - &client_run_integration_tests - run: - name: Install requirements and run integration tests - command: | - set -e - cd client - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test-integration - - - &client_run_functional_tests - run: - name: Install requirements and run functional tests - command: | - set -e - cd client - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test-functional - - - &client_run_lint - run: - name: Run lint, type checking, code formatting - command: | - set -e - make -C client check-black check-isort lint mypy - - - &client_check_security - run: - name: Run static analysis on source code to find security issues - command: | - set -e - make -C client semgrep - - - &client_check_source_strings - run: - name: Check that source strings are updated - command: | - set -e - make -C client check-strings - - - &client_check_mo_repro - run: - name: Check that translation machine objects are reproducible - command: | - set -e - make -C client verify-mo - - - &export_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &export_install_testing_dependencies - run: - name: Install testing dependencies - command: | - apt update && apt install -y git gnupg make python3-dev gnupg python3-venv libnotify-bin - cd export - poetry install --no-ansi - - - &export_run_unit_tests - run: - name: Install requirements and run unit tests - command: | - cd export - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test - - - &export_run_lint - run: - name: Run lint, type checking, code formatting - command: | - make -C export check-black lint - - - &export_check_security - run: - name: Run static analysis on source code to find security issues - command: | - make -C export semgrep - - - &log_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &log_install_testing_dependencies - run: - name: Install testing dependencies - command: | - apt-get install --yes --no-install-recommends git gnupg make - cd log - poetry install --no-ansi - - - &log_run_tests - run: - name: Install requirements and run tests - command: | - make -C log check - - - &proxy_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &proxy_install_testing_dependencies - run: - name: Install testing dependencies - command: | - apt-get install --yes --no-install-recommends git gnupg make - cd proxy - poetry install --no-ansi - - - &proxy_run_unit_tests - run: - name: Install requirements and run unit tests - command: | - cd proxy - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test - - - &proxy_run_lint - run: - name: Run lint, type checking, code formatting - command: | - make -C proxy lint - -version: 2.1 - -jobs: - client_unit-test: - parameters: ¶meters - image: - type: string - docker: &docker - - image: debian:<< parameters.image >> - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_configure_locales - - *client_run_unit_tests - - store_test_results: - path: test-results - - client_integration-test: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_run_integration_tests - - client_functional-test: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_run_functional_tests - - client_lint: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_run_lint - - client_check-security: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_check_security - - client_check-internationalization: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_check_source_strings - - *client_check_mo_repro - - export_unit-test: - parameters: *parameters - docker: *docker - steps: - - *export_install_poetry - - checkout - - *export_install_testing_dependencies - - *export_run_unit_tests - - store_test_results: - path: test-results - - export_lint: - parameters: *parameters - docker: *docker - steps: - - *export_install_poetry - - checkout - - *export_install_testing_dependencies - - *export_run_lint - - export_check-security: - parameters: *parameters - docker: *docker - steps: - - *export_install_poetry - - checkout - - *export_install_testing_dependencies - - *export_check_security - - log_test-bullseye: - docker: - - image: debian:bullseye - steps: - - checkout - - *log_install_poetry - - *log_install_testing_dependencies - - *log_run_tests - - proxy_unit-test: - parameters: *parameters - docker: *docker - steps: - - checkout - - *proxy_install_poetry - - *proxy_install_testing_dependencies - - *proxy_run_unit_tests - - store_test_results: - path: test-results - - proxy_lint: - parameters: *parameters - docker: *docker - steps: - - checkout - - *proxy_install_poetry - - *proxy_install_testing_dependencies - - *proxy_run_lint - - -workflows: - securedrop_client_ci: - jobs: &client_jobs - - client_unit-test: - matrix: &matrix - parameters: - image: - - bullseye - - bookworm - - client_integration-test: - matrix: *matrix - - client_functional-test: - matrix: *matrix - - client_lint: - matrix: *matrix - - client_check-security: - matrix: *matrix - - client_check-internationalization: - matrix: *matrix - - securedrop_export_ci: - jobs: &export_jobs - - export_unit-test: - matrix: *matrix - - export_lint: - matrix: *matrix - - export_check-security: - matrix: *matrix - - securedrop_log_ci: - jobs: - - log_test-bullseye - - securedrop_proxy_ci: - jobs: &proxy_jobs - - proxy_unit-test: - # bookworm jobs are failing and will be - # replaced with proxy v2 shortly, so skip - # https://github.com/freedomofpress/securedrop-client/issues/1681 - matrix: &proxy_matrix - parameters: - image: - - bullseye - - proxy_lint: - matrix: *proxy_matrix - - client_nightly: - triggers: - - schedule: - cron: "0 6 * * *" - filters: - branches: - only: - - main - jobs: *client_jobs - - export_nightly: - triggers: - - schedule: - cron: "0 6 * * *" - filters: - branches: - only: - - main - jobs: *export_jobs - - proxy_nightly: - triggers: - - schedule: - cron: "0 6 * * *" - filters: - branches: - only: - - main - jobs: *proxy_jobs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c4b2cc38..01da7bf6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,52 @@ jobs: - name: Run lint run: make lint + # Run `make lint` across all components + component-lint: + strategy: + fail-fast: false + matrix: + component: + - client + - export + - log + - proxy + debian_version: + - bullseye + - bookworm + # bookworm jobs are failing and will be + # replaced with proxy v2 shortly, so skip + # https://github.com/freedomofpress/securedrop-client/issues/1681 + exclude: + - component: proxy + debian_version: bookworm + runs-on: ubuntu-latest + container: debian:${{ matrix.debian_version }} + steps: + - run: | + apt-get update && apt-get install --yes git make gnupg + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + source /etc/os-release + if [[ "$VERSION_CODENAME" == "bullseye" ]]; then + # Install Poetry via PyPI + apt-get install --yes --no-install-recommends python3-pip + pip install poetry==1.6.1 + elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then + # Install Poetry via system package + apt-get install --yes --no-install-recommends python3-poetry + else + echo "Unsupported Debian version: $VERSION_CODENAME" + exit 1 + fi + poetry -C ${{ matrix.component }} install + if [[ "${{ matrix.component }}" == "client" ]]; then + make -C ${{ matrix.component }} ci-install-deps + fi + - name: Run lint + run: make -C ${{ matrix.component }} lint + safety: runs-on: ubuntu-latest container: debian:bookworm diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..55a673c06 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,124 @@ +name: Tests +on: [push, pull_request] + +defaults: + run: + shell: bash + +jobs: + # Run `make test` against all components but client, which is special + component: + strategy: + fail-fast: false + matrix: + component: + - export + - log + - proxy + debian_version: + - bullseye + - bookworm + # bookworm jobs are failing and will be + # replaced with proxy v2 shortly, so skip + # https://github.com/freedomofpress/securedrop-client/issues/1681 + exclude: + - component: proxy + debian_version: bookworm + runs-on: ubuntu-latest + container: debian:${{ matrix.debian_version }} + steps: + - run: | + apt-get update && apt-get install --yes git make gnupg + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + source /etc/os-release + if [[ "$VERSION_CODENAME" == "bullseye" ]]; then + # Install Poetry via PyPI + apt-get install --yes --no-install-recommends python3-pip + pip install poetry==1.6.1 + elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then + # Install Poetry via system package + apt-get install --yes --no-install-recommends python3-poetry + else + echo "Unsupported Debian version: $VERSION_CODENAME" + exit 1 + fi + poetry -C ${{ matrix.component }} install + - name: Run test + run: | + make -C ${{ matrix.component }} test + + # Run the various `make test-...` commands for the client. + # TODO: these should be consolidated into one when feasible + client: + strategy: + fail-fast: false + matrix: + command: + - test-functional + - test-integration + - test-random + debian_version: + - bullseye + - bookworm + runs-on: ubuntu-latest + container: debian:${{ matrix.debian_version }} + steps: + - run: | + apt-get update && apt-get install --yes git make gnupg + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + source /etc/os-release + if [[ "$VERSION_CODENAME" == "bullseye" ]]; then + # Install Poetry via PyPI + apt-get install --yes --no-install-recommends python3-pip + pip install poetry==1.6.1 + elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then + # Install Poetry via system package + apt-get install --yes --no-install-recommends python3-poetry + else + echo "Unsupported Debian version: $VERSION_CODENAME" + exit 1 + fi + poetry -C client install + make -C client ci-install-deps + - name: Run test + run: | + make -C client ${{ matrix.command }} + + # Run the client i18n/l10n checks. + internationalization: + strategy: + matrix: + debian_version: + - bullseye + - bookworm + runs-on: ubuntu-latest + container: debian:${{ matrix.debian_version }} + steps: + - run: | + apt-get update && apt-get install --yes git make + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + source /etc/os-release + if [[ "$VERSION_CODENAME" == "bullseye" ]]; then + # Install Poetry via PyPI + apt-get install --yes --no-install-recommends python3-pip + pip install poetry==1.6.1 + elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then + # Install Poetry via system package + apt-get install --yes --no-install-recommends python3-poetry + else + echo "Unsupported Debian version: $VERSION_CODENAME" + exit 1 + fi + poetry -C client install + make -C client ci-install-deps + git config --global --add safe.directory '*' + - name: Check that source strings are updated + run: make -C client check-strings + - name: Check that translation machine objects are reproducible + run: make -C client verify-mo diff --git a/client/Makefile b/client/Makefile index afa117fc8..baff5ce67 100644 --- a/client/Makefile +++ b/client/Makefile @@ -189,3 +189,12 @@ verify-mo: ## Verify that all gettext machine objects (.mo) are reproducible fro @TERM=dumb poetry run scripts/verify-mo.py ${LOCALE_DIR}/* @# All good; now clean up. @git restore "${LOCALE_DIR}/**/*.po" + +# Install dependencies in CI +# Note, we don't actually need libqt5x11extras5, but it pulls in all +# the correct dependencies for the PyQt5 wheels. +ci-install-deps: + apt-get install --yes xvfb sqlite3 locales libqt5x11extras5 python3-tk + echo "en_US ISO-8859-1" >> /etc/locale.gen + echo "en_US UTF-8" >> /etc/locale.gen + locale-gen