diff --git a/.github/workflows/instrumentations_0.yml b/.github/workflows/instrumentations_0.yml index d85102f435..d54cb50119 100644 --- a/.github/workflows/instrumentations_0.yml +++ b/.github/workflows/instrumentations_0.yml @@ -126,5 +126,5 @@ jobs: .tox ~/.cache/pip key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }} - - name: run pytest without --benchmark-skip + - name: run tox run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra diff --git a/.github/workflows/instrumentations_1.yml b/.github/workflows/instrumentations_1.yml index af311343df..8c99eb0572 100644 --- a/.github/workflows/instrumentations_1.yml +++ b/.github/workflows/instrumentations_1.yml @@ -25,7 +25,6 @@ jobs: matrix: python-version: [py38, py39, py310, py311, py312, pypy3] package: - # Only add here packages that do not have benchmark tests - "urllib" - "urllib3" - "wsgi" @@ -33,6 +32,8 @@ jobs: - "richconsole" - "psycopg" - "prometheus-remote-write" + - "sdk-extension-aws" + - "propagator-aws-xray" - "propagator-ot-trace" - "resource-detector-azure" - "resource-detector-container" @@ -58,5 +59,5 @@ jobs: .tox ~/.cache/pip key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }} - - name: run pytest without --benchmark-skip + - name: run tox run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra diff --git a/.github/workflows/instrumentations_2.yml b/.github/workflows/instrumentations_2.yml deleted file mode 100644 index 48e7e127c8..0000000000 --- a/.github/workflows/instrumentations_2.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Contrib Repo Tests - -on: - push: - branches-ignore: - - 'release/*' - pull_request: -env: - CORE_REPO_SHA: 141a6a2e473ef7f0ec4915dfb71e3c0fa595283e - -jobs: - instrumentations-2: - env: - # We use these variables to convert between tox and GHA version literals - py38: 3.8 - py39: 3.9 - py310: "3.10" - py311: "3.11" - py312: "3.12" - pypy3: pypy-3.8 - RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false # ensures the entire test matrix is run, even if one permutation fails - matrix: - python-version: [py38, py39, py310, py311, py312, pypy3] - package: - # Only add here packages that have benchmark tests - - "sdk-extension-aws" - - "propagator-aws-xray" - os: [ubuntu-20.04] - steps: - - name: Checkout Contrib Repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - name: Set up Python ${{ env[matrix.python-version] }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env[matrix.python-version] }} - - name: Install tox - run: pip install tox - - name: Cache tox environment - # Preserves .tox directory between runs for faster installs - uses: actions/cache@v4 - with: - path: | - .tox - ~/.cache/pip - key: v7-build-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'gen-requirements.txt', 'dev-requirements.txt') }} - - name: run pytest with --benchmark-skip - run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra --benchmark-skip diff --git a/.gitignore b/.gitignore index c359e13727..1c32b4446a 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,6 @@ _build/ # mypy .mypy_cache/ target + +# Benchmark result files +*-benchmark.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f5a4913440..d7fc509c3a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,9 +96,7 @@ for more detail on available tox commands. ### Benchmarks -Performance progression of benchmarks for packages distributed by OpenTelemetry Python can be viewed as a [graph of throughput vs commit history](https://opentelemetry-python-contrib.readthedocs.io/en/latest/performance/benchmarks.html). From the linked page, you can download a JSON file with the performance results. - -Running the `tox` tests also runs the performance tests if any are available. Benchmarking tests are done with `pytest-benchmark` and they output a table with results to the console. +Some packages have benchmark tests. To run them, run `tox -f benchmark`. Benchmark tests use `pytest-benchmark` and they output a table with results to the console. To write benchmarks, simply use the [pytest benchmark fixture](https://pytest-benchmark.readthedocs.io/en/latest/usage.html#usage) like the following: @@ -114,10 +112,10 @@ def test_simple_start_span(benchmark): benchmark(benchmark_start_as_current_span, "benchmarkedSpan", 42) ``` -Make sure the test file is under the `tests/performance/benchmarks/` folder of +Make sure the test file is under the `benchmarks/` folder of the package it is benchmarking and further has a path that corresponds to the file in the package it is testing. Make sure that the file name begins with -`test_benchmark_`. (e.g. `propagator/opentelemetry-propagator-aws-xray/tests/performance/benchmarks/trace/propagation/test_benchmark_aws_xray_propagator.py`) +`test_benchmark_`. (e.g. `propagator/opentelemetry-propagator-aws-xray/benchmarks/trace/propagation/test_benchmark_aws_xray_propagator.py`) ## Pull Requests diff --git a/propagator/opentelemetry-propagator-aws-xray/benchmark-requirements.txt b/propagator/opentelemetry-propagator-aws-xray/benchmark-requirements.txt new file mode 100644 index 0000000000..44564857ef --- /dev/null +++ b/propagator/opentelemetry-propagator-aws-xray/benchmark-requirements.txt @@ -0,0 +1 @@ +pytest-benchmark==4.0.0 diff --git a/propagator/opentelemetry-propagator-aws-xray/tests/performance/benchmarks/test_benchmark_aws_xray_propagator.py b/propagator/opentelemetry-propagator-aws-xray/benchmarks/test_benchmark_aws_xray_propagator.py similarity index 100% rename from propagator/opentelemetry-propagator-aws-xray/tests/performance/benchmarks/test_benchmark_aws_xray_propagator.py rename to propagator/opentelemetry-propagator-aws-xray/benchmarks/test_benchmark_aws_xray_propagator.py diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/benchmark-requirements.txt b/sdk-extension/opentelemetry-sdk-extension-aws/benchmark-requirements.txt new file mode 100644 index 0000000000..44564857ef --- /dev/null +++ b/sdk-extension/opentelemetry-sdk-extension-aws/benchmark-requirements.txt @@ -0,0 +1 @@ +pytest-benchmark==4.0.0 diff --git a/sdk-extension/opentelemetry-sdk-extension-aws/tests/performance/benchmarks/trace/test_benchmark_aws_xray_ids_generator.py b/sdk-extension/opentelemetry-sdk-extension-aws/benchmarks/trace/test_benchmark_aws_xray_ids_generator.py similarity index 100% rename from sdk-extension/opentelemetry-sdk-extension-aws/tests/performance/benchmarks/trace/test_benchmark_aws_xray_ids_generator.py rename to sdk-extension/opentelemetry-sdk-extension-aws/benchmarks/trace/test_benchmark_aws_xray_ids_generator.py diff --git a/tox.ini b/tox.ini index 4399ccc1e1..33f0242c5a 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ envlist = py3{8,9,10,11,12}-test-sdk-extension-aws pypy3-test-sdk-extension-aws lint-sdk-extension-aws + benchmark-sdk-extension-aws ; opentelemetry-distro py3{8,9,10,11,12}-test-distro @@ -314,6 +315,7 @@ envlist = py3{8,9,10,11,12}-test-propagator-aws-xray pypy3-test-propagator-aws-xray lint-propagator-aws-xray + benchmark-propagator-aws-xray ; opentelemetry-propagator-ot-trace py3{8,9,10,11,12}-test-propagator-ot-trace @@ -741,6 +743,7 @@ commands_pre = sdk-extension-aws: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk sdk-extension-aws: pip install opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils sdk-extension-aws: pip install -r {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws/test-requirements.txt + benchmark-sdk-extension-aws: pip install -r {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws/benchmark-requirements.txt resource-detector-container: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api resource-detector-container: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions @@ -765,6 +768,7 @@ commands_pre = propagator-aws-xray: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk propagator-aws-xray: pip install opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils propagator-aws-xray: pip install -r {toxinidir}/propagator/opentelemetry-propagator-aws-xray/test-requirements.txt + benchmark-propagator-aws-xray: pip install -r {toxinidir}/propagator/opentelemetry-propagator-aws-xray/benchmark-requirements.txt processor-baggage: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api processor-baggage: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions @@ -1094,6 +1098,7 @@ commands = lint-sdk-extension-aws: isort --diff --check-only --settings-path {toxinidir}/.isort.cfg {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws lint-sdk-extension-aws: flake8 --config {toxinidir}/.flake8 {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws lint-sdk-extension-aws: sh -c "cd sdk-extension && pylint --rcfile ../.pylintrc opentelemetry-sdk-extension-aws" + benchmark-sdk-extension-aws: pytest {toxinidir}/sdk-extension/opentelemetry-sdk-extension-aws/benchmarks {posargs} --benchmark-json=sdk-extension-aws-benchmark.json test-resource-detector-container: pytest {toxinidir}/resource/opentelemetry-resource-detector-container/tests {posargs} lint-resource-detector-container: black --diff --check --config {toxinidir}/pyproject.toml {toxinidir}/resource/opentelemetry-resource-detector-container @@ -1118,6 +1123,7 @@ commands = lint-propagator-aws-xray: isort --diff --check-only --settings-path {toxinidir}/.isort.cfg {toxinidir}/propagator/opentelemetry-propagator-aws-xray lint-propagator-aws-xray: flake8 --config {toxinidir}/.flake8 {toxinidir}/propagator/opentelemetry-propagator-aws-xray lint-propagator-aws-xray: sh -c "cd propagator && pylint --rcfile ../.pylintrc opentelemetry-propagator-aws-xray" + benchmark-propagator-aws-xray: pytest {toxinidir}/propagator/opentelemetry-propagator-aws-xray/benchmarks {posargs} --benchmark-json=propagator-aws-xray-benchmark.json test-propagator-ot-trace: pytest {toxinidir}/propagator/opentelemetry-propagator-ot-trace/tests {posargs} lint-propagator-ot-trace: black --diff --check --config {toxinidir}/pyproject.toml {toxinidir}/propagator/opentelemetry-propagator-ot-trace