Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make nox test name sessions more logical #1135

Merged
merged 2 commits into from
Sep 29, 2022
Merged

Make nox test name sessions more logical #1135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
24cec5a
Make nox test name sessions more logical
ThomasLaPiana Sep 29, 2022
0859b09
consolidate the nox test sessions
ThomasLaPiana Sep 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .github/workflows/backend_checks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ jobs:
run: pip install nox>=2022

- name: Run non-external test suite
run: nox -s "pytest(not-external)"
run: nox -s "pytest_ctl(not-external)"


Pytest-Unit-Ops:
Expand All @@ -246,7 +246,7 @@ jobs:
run: pip install nox>=2022

- name: Run unit test suite
run: nox -s pytest_unit
run: nox -s "pytest_ops(unit)"

Pytest-Integration-Ops:
needs: Build
Expand All @@ -272,7 +272,7 @@ jobs:
run: pip install nox>=2022

- name: Run integration test suite
run: nox -s pytest_integration
run: nox -s "pytest_ops(integration)"

##################
## Unsafe Tests ##
Expand Down Expand Up @@ -303,7 +303,7 @@ jobs:
run: pip install nox>=2022

- name: Run external test suite
run: nox -s pytest_external
run: nox -s "pytest_ctl(external)"
env:
SNOWFLAKE_FIDESCTL_PASSWORD: ${{ secrets.SNOWFLAKE_FIDESCTL_PASSWORD }}
REDSHIFT_FIDESCTL_PASSWORD: ${{ secrets.REDSHIFT_FIDESCTL_PASSWORD }}
Expand Down Expand Up @@ -345,7 +345,7 @@ jobs:
BIGQUERY_KEYFILE_CREDS: ${{ secrets.BIGQUERY_KEYFILE_CREDS }}
BIGQUERY_DATASET: fidesopstest
SNOWFLAKE_TEST_URI: ${{ secrets.SNOWFLAKE_TEST_URI }}
run: nox -s pytest_integration_external
run: nox -s "pytest_ops(external-datastores)"

External-SaaS-Connectors:
needs: Build
Expand Down Expand Up @@ -388,4 +388,4 @@ jobs:
env:
VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
VAULT_NAMESPACE: ${{ secrets.VAULT_NAMESPACE }}
run: nox -s pytest_saas
run: nox -s "pytest_ops(saas)"
281 changes: 126 additions & 155 deletions noxfiles/ci_nox.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,28 +17,6 @@

RUN_STATIC_ANALYSIS = (*RUN_NO_DEPS, "nox", "-s")


@nox.session()
def ci_suite(session: nox.Session) -> None:
"""
Runs the CI check suite.

Excludes external tests so that no additional secrets/tooling are required.
"""
# Use "notify" instead of direct calls here to provide better user feedback
session.notify("teardown")
session.notify("build", ["test"])
session.notify("black")
session.notify("isort")
session.notify("xenon")
session.notify("mypy")
session.notify("pylint")
session.notify("check_install")
session.notify("pytest_unit")
session.notify("pytest_integration")
session.notify("teardown")


# Static Checks
@nox.session()
def static_checks(session: nox.Session) -> None:
Expand Down Expand Up @@ -150,147 +128,140 @@ def fides_db_scan(session: nox.Session) -> None:
nox.param("unit", id="unit"),
nox.param("integration", id="integration"),
nox.param("not external", id="not-external"),
nox.param("external", id="external"),
],
)
def pytest(session: nox.Session, mark: str) -> None:
def pytest_ctl(session: nox.Session, mark: str) -> None:
"""Runs fidesctl tests."""
session.notify("teardown")
session.run(*START_APP, external=True)
run_command = (
*RUN_NO_DEPS,
"pytest",
"tests/ctl/",
"-m",
mark,
)
session.run(*run_command, external=True)


@nox.session()
def pytest_unit(session: nox.Session) -> None:
"""Runs fidesops unit tests."""
session.notify("teardown")
session.run(*START_APP, external=True)
run_command = (
*RUN_NO_DEPS,
"pytest",
OPS_TEST_DIR,
"-m",
"not integration and not integration_external and not integration_saas",
)
session.run(*run_command, external=True)


@nox.session()
def pytest_external(session: nox.Session) -> None:
"""Run all fidesctl tests that rely on the third-party databases and services."""
start_command = (
"docker-compose",
"-f",
COMPOSE_FILE,
"-f",
INTEGRATION_COMPOSE_FILE,
"up",
"-d",
IMAGE_NAME,
)
session.run(*start_command, external=True)
run_command = (
"docker-compose",
"run",
"-e",
"SNOWFLAKE_FIDESCTL_PASSWORD",
"-e",
"REDSHIFT_FIDESCTL_PASSWORD",
"-e",
"AWS_ACCESS_KEY_ID",
"-e",
"AWS_SECRET_ACCESS_KEY",
"-e",
"AWS_DEFAULT_REGION",
"-e",
"OKTA_CLIENT_TOKEN",
"-e",
"BIGQUERY_CONFIG",
"--rm",
CI_ARGS,
IMAGE_NAME,
"pytest",
"-m",
"external",
"tests/ctl",
)
session.run(*run_command, external=True)


@nox.session()
def pytest_integration(session: nox.Session) -> None:
"""Runs fidesops integration tests."""
session.notify("teardown")
run_infrastructure(
run_tests=True,
analytics_opt_out=True,
datastores=[],
pytest_path=OPS_TEST_DIR,
)


@nox.session()
def pytest_integration_external(session: nox.Session) -> None:
"""Run all tests that rely on the third-party databases and services."""
session.notify("teardown")
session.run(*START_APP, external=True)
run_command = (
"docker",
"compose",
"run",
"-e",
"ANALYTICS_OPT_OUT",
"-e",
"REDSHIFT_TEST_URI",
"-e",
"SNOWFLAKE_TEST_URI",
"-e",
"REDSHIFT_TEST_DB_SCHEMA",
"-e",
"BIGQUERY_KEYFILE_CREDS",
"-e",
"BIGQUERY_DATASET",
"--rm",
CI_ARGS,
COMPOSE_SERVICE_NAME,
"pytest",
OPS_TEST_DIR,
"-m",
"integration_external",
)
session.run(*run_command, external=True)
if mark == "external":
start_command = (
"docker-compose",
"-f",
COMPOSE_FILE,
"-f",
INTEGRATION_COMPOSE_FILE,
"up",
"-d",
IMAGE_NAME,
)
session.run(*start_command, external=True)
run_command = (
"docker-compose",
"run",
"-e",
"SNOWFLAKE_FIDESCTL_PASSWORD",
"-e",
"REDSHIFT_FIDESCTL_PASSWORD",
"-e",
"AWS_ACCESS_KEY_ID",
"-e",
"AWS_SECRET_ACCESS_KEY",
"-e",
"AWS_DEFAULT_REGION",
"-e",
"OKTA_CLIENT_TOKEN",
"-e",
"BIGQUERY_CONFIG",
"--rm",
CI_ARGS,
IMAGE_NAME,
"pytest",
"-m",
"external",
"tests/ctl",
)
session.run(*run_command, external=True)
else:
session.run(*START_APP, external=True)
run_command = (
*RUN_NO_DEPS,
"pytest",
"tests/ctl/",
"-m",
mark,
)
session.run(*run_command, external=True)


@nox.session()
def pytest_saas(session: nox.Session) -> None:
"""Run all saas tests that rely on the third-party databases and services."""
@nox.parametrize(
"mark",
[
nox.param("unit", id="unit"),
nox.param("integration", id="integration"),
nox.param("external_datastores", id="external-datastores"),
nox.param("saas", id="saas"),
],
)
def pytest_ops(session: nox.Session, mark: str) -> None:
"""Runs fidesops tests."""
session.notify("teardown")
run_command = (
"docker-compose",
"run",
"-e",
"ANALYTICS_OPT_OUT",
"-e",
"VAULT_ADDR",
"-e",
"VAULT_NAMESPACE",
"-e",
"VAULT_TOKEN",
"--rm",
CI_ARGS,
COMPOSE_SERVICE_NAME,
"pytest",
OPS_TEST_DIR,
"-m",
"integration_saas",
)
session.run(*run_command, external=True)
if mark == "unit":
session.run(*START_APP, external=True)
run_command = (
*RUN_NO_DEPS,
"pytest",
OPS_TEST_DIR,
"-m",
"not integration and not integration_external and not integration_saas",
)
session.run(*run_command, external=True)
elif mark == "integration":
run_infrastructure(
run_tests=True,
analytics_opt_out=True,
datastores=[],
pytest_path=OPS_TEST_DIR,
)
elif mark == "external_datastores":
session.run(*START_APP, external=True)
run_command = (
"docker",
"compose",
"run",
"-e",
"ANALYTICS_OPT_OUT",
"-e",
"REDSHIFT_TEST_URI",
"-e",
"SNOWFLAKE_TEST_URI",
"-e",
"REDSHIFT_TEST_DB_SCHEMA",
"-e",
"BIGQUERY_KEYFILE_CREDS",
"-e",
"BIGQUERY_DATASET",
"--rm",
CI_ARGS,
COMPOSE_SERVICE_NAME,
"pytest",
OPS_TEST_DIR,
"-m",
"integration_external",
)
session.run(*run_command, external=True)
elif mark == "saas":
run_command = (
"docker-compose",
"run",
"-e",
"ANALYTICS_OPT_OUT",
"-e",
"VAULT_ADDR",
"-e",
"VAULT_NAMESPACE",
"-e",
"VAULT_TOKEN",
"--rm",
CI_ARGS,
COMPOSE_SERVICE_NAME,
"pytest",
OPS_TEST_DIR,
"-m",
"integration_saas",
)
session.run(*run_command, external=True)


@nox.session()
Expand Down
19 changes: 1 addition & 18 deletions noxfiles/dev_nox.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Contains the nox sessions for running development environments."""
import nox

from constants_nox import ANALYTICS_OPT_OUT, COMPOSE_SERVICE_NAME, RUN, START_APP
from constants_nox import COMPOSE_SERVICE_NAME, RUN, START_APP
from docker_nox import build
from run_infrastructure import ALL_DATASTORES, run_infrastructure

Expand Down Expand Up @@ -34,23 +34,6 @@ def dev(session: nox.Session) -> None:
)


@nox.session()
def dev_with_worker(session: nox.Session) -> None:
"""Spin up the entire application and open a development shell."""
build(session, "dev")
session.notify("teardown")
session.run("docker", "compose", "up", "worker", "--wait", external=True)
session.run(
"docker-compose",
"run",
*ANALYTICS_OPT_OUT,
"-e",
"FIDES__EXECUTION__WORKER_ENABLED=True",
COMPOSE_SERVICE_NAME,
external=True,
)


@nox.session()
def quickstart(session: nox.Session) -> None:
"""Run the quickstart tutorial."""
Expand Down