diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 26e139e0e..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,410 +0,0 @@ ---- -common-steps: - - &client_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &client_install_testing_dependencies - run: - name: Install testing dependencies - command: | - set -e - apt update && apt install -y git gnupg libarchive13 libmagic1 libqt5x11extras5 make python3-tk python3-dev gnupg python3-venv sqlite3 xvfb - cd client - poetry install --no-ansi - - - &client_configure_locales - run: - name: Configure locales - command: | - set -e - apt update && apt install -y locales - echo "en_US ISO-8859-1" >> /etc/locale.gen - echo "en_US UTF-8" >> /etc/locale.gen - locale-gen - - - &client_run_unit_tests - run: - name: Install requirements and run unit tests - command: | - set -e - cd client - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test-random - - - &client_run_integration_tests - run: - name: Install requirements and run integration tests - command: | - set -e - cd client - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test-integration - - - &client_run_functional_tests - run: - name: Install requirements and run functional tests - command: | - set -e - cd client - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test-functional - - - &client_run_lint - run: - name: Run lint, type checking, code formatting - command: | - set -e - make -C client check-black check-isort lint mypy - - - &client_check_security - run: - name: Run static analysis on source code to find security issues - command: | - set -e - make -C client semgrep - - - &client_check_source_strings - run: - name: Check that source strings are updated - command: | - set -e - make -C client check-strings - - - &client_check_mo_repro - run: - name: Check that translation machine objects are reproducible - command: | - set -e - make -C client verify-mo - - - &export_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &export_install_testing_dependencies - run: - name: Install testing dependencies - command: | - apt update && apt install -y git gnupg make python3-dev gnupg python3-venv libnotify-bin - cd export - poetry install --no-ansi - - - &export_run_unit_tests - run: - name: Install requirements and run unit tests - command: | - cd export - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test - - - &export_run_lint - run: - name: Run lint, type checking, code formatting - command: | - make -C export check-black lint - - - &export_check_security - run: - name: Run static analysis on source code to find security issues - command: | - make -C export semgrep - - - &log_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &log_install_testing_dependencies - run: - name: Install testing dependencies - command: | - apt-get install --yes --no-install-recommends git gnupg make - cd log - poetry install --no-ansi - - - &log_run_tests - run: - name: Install requirements and run tests - command: | - make -C log check - - - &proxy_install_poetry - run: - name: Install Poetry - command: | - set -e - source /etc/os-release - if [[ "$VERSION_CODENAME" == "bullseye" ]]; then - # Install Poetry via PyPI - apt-get update && apt-get install --yes --no-install-recommends python3-pip - pip install poetry==1.6.1 - elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then - # Install Poetry via system package - apt-get update && apt-get install --yes --no-install-recommends python3-poetry - else - echo "Unsupported Debian version: $VERSION_CODENAME" - exit 1 - fi - - - &proxy_install_testing_dependencies - run: - name: Install testing dependencies - command: | - apt-get install --yes --no-install-recommends git gnupg make - cd proxy - poetry install --no-ansi - - - &proxy_run_unit_tests - run: - name: Install requirements and run unit tests - command: | - cd proxy - export PYTHONPATH=$PYTHONPATH:. # so alembic can get to Base metadata - make test - - - &proxy_run_lint - run: - name: Run lint, type checking, code formatting - command: | - make -C proxy lint - -version: 2.1 - -jobs: - client_unit-test: - parameters: ¶meters - image: - type: string - docker: &docker - - image: debian:<< parameters.image >> - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_configure_locales - - *client_run_unit_tests - - store_test_results: - path: test-results - - client_integration-test: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_run_integration_tests - - client_functional-test: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_run_functional_tests - - client_lint: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_run_lint - - client_check-security: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_check_security - - client_check-internationalization: - parameters: *parameters - docker: *docker - steps: - - *client_install_poetry - - checkout - - *client_install_testing_dependencies - - *client_check_source_strings - - *client_check_mo_repro - - export_unit-test: - parameters: *parameters - docker: *docker - steps: - - *export_install_poetry - - checkout - - *export_install_testing_dependencies - - *export_run_unit_tests - - store_test_results: - path: test-results - - export_lint: - parameters: *parameters - docker: *docker - steps: - - *export_install_poetry - - checkout - - *export_install_testing_dependencies - - *export_run_lint - - export_check-security: - parameters: *parameters - docker: *docker - steps: - - *export_install_poetry - - checkout - - *export_install_testing_dependencies - - *export_check_security - - log_test-bullseye: - docker: - - image: debian:bullseye - steps: - - checkout - - *log_install_poetry - - *log_install_testing_dependencies - - *log_run_tests - - proxy_unit-test: - parameters: *parameters - docker: *docker - steps: - - checkout - - *proxy_install_poetry - - *proxy_install_testing_dependencies - - *proxy_run_unit_tests - - store_test_results: - path: test-results - - proxy_lint: - parameters: *parameters - docker: *docker - steps: - - checkout - - *proxy_install_poetry - - *proxy_install_testing_dependencies - - *proxy_run_lint - - -workflows: - securedrop_client_ci: - jobs: &client_jobs - - client_unit-test: - matrix: &matrix - parameters: - image: - - bullseye - - bookworm - - client_integration-test: - matrix: *matrix - - client_functional-test: - matrix: *matrix - - client_lint: - matrix: *matrix - - client_check-security: - matrix: *matrix - - client_check-internationalization: - matrix: *matrix - - securedrop_export_ci: - jobs: &export_jobs - - export_unit-test: - matrix: *matrix - - export_lint: - matrix: *matrix - - export_check-security: - matrix: *matrix - - securedrop_log_ci: - jobs: - - log_test-bullseye - - securedrop_proxy_ci: - jobs: &proxy_jobs - - proxy_unit-test: - # bookworm jobs are failing and will be - # replaced with proxy v2 shortly, so skip - # https://github.com/freedomofpress/securedrop-client/issues/1681 - matrix: &proxy_matrix - parameters: - image: - - bullseye - - proxy_lint: - matrix: *proxy_matrix - - client_nightly: - triggers: - - schedule: - cron: "0 6 * * *" - filters: - branches: - only: - - main - jobs: *client_jobs - - export_nightly: - triggers: - - schedule: - cron: "0 6 * * *" - filters: - branches: - only: - - main - jobs: *export_jobs - - proxy_nightly: - triggers: - - schedule: - cron: "0 6 * * *" - filters: - branches: - only: - - main - jobs: *proxy_jobs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c4b2cc38..01da7bf6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,52 @@ jobs: - name: Run lint run: make lint + # Run `make lint` across all components + component-lint: + strategy: + fail-fast: false + matrix: + component: + - client + - export + - log + - proxy + debian_version: + - bullseye + - bookworm + # bookworm jobs are failing and will be + # replaced with proxy v2 shortly, so skip + # https://github.com/freedomofpress/securedrop-client/issues/1681 + exclude: + - component: proxy + debian_version: bookworm + runs-on: ubuntu-latest + container: debian:${{ matrix.debian_version }} + steps: + - run: | + apt-get update && apt-get install --yes git make gnupg + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + source /etc/os-release + if [[ "$VERSION_CODENAME" == "bullseye" ]]; then + # Install Poetry via PyPI + apt-get install --yes --no-install-recommends python3-pip + pip install poetry==1.6.1 + elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then + # Install Poetry via system package + apt-get install --yes --no-install-recommends python3-poetry + else + echo "Unsupported Debian version: $VERSION_CODENAME" + exit 1 + fi + poetry -C ${{ matrix.component }} install + if [[ "${{ matrix.component }}" == "client" ]]; then + make -C ${{ matrix.component }} ci-install-deps + fi + - name: Run lint + run: make -C ${{ matrix.component }} lint + safety: runs-on: ubuntu-latest container: debian:bookworm diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..55a673c06 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,124 @@ +name: Tests +on: [push, pull_request] + +defaults: + run: + shell: bash + +jobs: + # Run `make test` against all components but client, which is special + component: + strategy: + fail-fast: false + matrix: + component: + - export + - log + - proxy + debian_version: + - bullseye + - bookworm + # bookworm jobs are failing and will be + # replaced with proxy v2 shortly, so skip + # https://github.com/freedomofpress/securedrop-client/issues/1681 + exclude: + - component: proxy + debian_version: bookworm + runs-on: ubuntu-latest + container: debian:${{ matrix.debian_version }} + steps: + - run: | + apt-get update && apt-get install --yes git make gnupg + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + source /etc/os-release + if [[ "$VERSION_CODENAME" == "bullseye" ]]; then + # Install Poetry via PyPI + apt-get install --yes --no-install-recommends python3-pip + pip install poetry==1.6.1 + elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then + # Install Poetry via system package + apt-get install --yes --no-install-recommends python3-poetry + else + echo "Unsupported Debian version: $VERSION_CODENAME" + exit 1 + fi + poetry -C ${{ matrix.component }} install + - name: Run test + run: | + make -C ${{ matrix.component }} test + + # Run the various `make test-...` commands for the client. + # TODO: these should be consolidated into one when feasible + client: + strategy: + fail-fast: false + matrix: + command: + - test-functional + - test-integration + - test-random + debian_version: + - bullseye + - bookworm + runs-on: ubuntu-latest + container: debian:${{ matrix.debian_version }} + steps: + - run: | + apt-get update && apt-get install --yes git make gnupg + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + source /etc/os-release + if [[ "$VERSION_CODENAME" == "bullseye" ]]; then + # Install Poetry via PyPI + apt-get install --yes --no-install-recommends python3-pip + pip install poetry==1.6.1 + elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then + # Install Poetry via system package + apt-get install --yes --no-install-recommends python3-poetry + else + echo "Unsupported Debian version: $VERSION_CODENAME" + exit 1 + fi + poetry -C client install + make -C client ci-install-deps + - name: Run test + run: | + make -C client ${{ matrix.command }} + + # Run the client i18n/l10n checks. + internationalization: + strategy: + matrix: + debian_version: + - bullseye + - bookworm + runs-on: ubuntu-latest + container: debian:${{ matrix.debian_version }} + steps: + - run: | + apt-get update && apt-get install --yes git make + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + source /etc/os-release + if [[ "$VERSION_CODENAME" == "bullseye" ]]; then + # Install Poetry via PyPI + apt-get install --yes --no-install-recommends python3-pip + pip install poetry==1.6.1 + elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then + # Install Poetry via system package + apt-get install --yes --no-install-recommends python3-poetry + else + echo "Unsupported Debian version: $VERSION_CODENAME" + exit 1 + fi + poetry -C client install + make -C client ci-install-deps + git config --global --add safe.directory '*' + - name: Check that source strings are updated + run: make -C client check-strings + - name: Check that translation machine objects are reproducible + run: make -C client verify-mo diff --git a/client/Makefile b/client/Makefile index 92d0485ac..baff5ce67 100644 --- a/client/Makefile +++ b/client/Makefile @@ -109,12 +109,15 @@ test-functional: ## Run the functional tests test-sdk: ## Run just the sdk tests @poetry run pytest -v $(STESTS) -.PHONY: lint -lint: ## Run the linters +.PHONY: flake8 +flake8: ## Run flake8 linting @poetry run flake8 securedrop_client tests +.PHONY: lint +lint: check-black check-isort flake8 mypy semgrep ## Run all linters + .PHONY: check -check: clean check-black check-isort semgrep lint mypy test-random test-integration test-functional ## Run the full CI test suite +check: clean lint test-random test-integration test-functional ## Run the full CI test suite # Explanation of the below shell command should it ever break. # 1. Set the field separator to ": ##" and any make targets that might appear between : and ## @@ -186,3 +189,12 @@ verify-mo: ## Verify that all gettext machine objects (.mo) are reproducible fro @TERM=dumb poetry run scripts/verify-mo.py ${LOCALE_DIR}/* @# All good; now clean up. @git restore "${LOCALE_DIR}/**/*.po" + +# Install dependencies in CI +# Note, we don't actually need libqt5x11extras5, but it pulls in all +# the correct dependencies for the PyQt5 wheels. +ci-install-deps: + apt-get install --yes xvfb sqlite3 locales libqt5x11extras5 python3-tk + echo "en_US ISO-8859-1" >> /etc/locale.gen + echo "en_US UTF-8" >> /etc/locale.gen + locale-gen diff --git a/export/Makefile b/export/Makefile index ad65eaade..ba88300a9 100644 --- a/export/Makefile +++ b/export/Makefile @@ -1,8 +1,12 @@ .PHONY: all all: help +.PHONY: lint +lint: check-black flake8 mypy semgrep ## Run all linters + + .PHONY: check -check: lint mypy semgrep test check-black ## Run linter and tests +check: lint test ## Run linter and tests .PHONY: check-black check-black: ## Check Python source code formatting with black @@ -13,8 +17,8 @@ TESTS ?= tests test: ## Run tests poetry run pytest -v --cov-report html --cov-report term-missing --cov=securedrop_export $$TESTS -.PHONY: lint -lint: ## Run linter +.PHONY: flake8 +flake8: ## Run flake8 linter poetry run flake8 securedrop_export/ tests/ .PHONY: mypy diff --git a/log/Makefile b/log/Makefile index 0d5febc56..79180d4cf 100644 --- a/log/Makefile +++ b/log/Makefile @@ -2,7 +2,10 @@ DEFAULT_GOAL: help SHELL := /bin/bash .PHONY: check -check: flake8 mypy test +check: lint test + +.PHONY: lint +lint: flake8 mypy ## Run all linters .PHONY: flake8 flake8: ## Run flake8 to lint Python files