Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[forge] Add junit support #15035

Merged
merged 1 commit into from
Oct 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/docker-build-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,7 @@ jobs:
# by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace.
FORGE_NAMESPACE: forge-e2e-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
SEND_RESULTS_TO_TRUNK: true

# This job determines the last released docker image tag, which is used by forge compat test.
fetch-last-released-docker-image-tag:
Expand Down Expand Up @@ -356,6 +357,7 @@ jobs:
COMMENT_HEADER: forge-compat
FORGE_NAMESPACE: forge-compat-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
SEND_RESULTS_TO_TRUNK: true

# Run forge framework upgradability test. This is a PR required job.
forge-framework-upgrade-test:
Expand Down Expand Up @@ -385,6 +387,7 @@ jobs:
COMMENT_HEADER: forge-framework-upgrade
FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
SKIP_JOB: ${{ !contains(github.event.pull_request.labels.*.name, 'CICD:run-framework-upgrade-test') && (needs.test-target-determinator.outputs.run_framework_upgrade_test == 'false') }}
SEND_RESULTS_TO_TRUNK: true

forge-consensus-only-perf-test:
needs:
Expand Down
16 changes: 16 additions & 0 deletions .github/workflows/forge-stable.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
FORGE_TEST_SUITE: framework_upgrade
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

run-forge-realistic-env-load-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
Expand All @@ -148,6 +149,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 1800 # Run for 30 minutes (6 tests, each for 300 seconds)
FORGE_TEST_SUITE: realistic_env_load_sweep
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

run-forge-realistic-env-workload-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
Expand All @@ -160,6 +162,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 2000 # Run for 33 minutes (5 tests, each for 400 seconds)
FORGE_TEST_SUITE: realistic_env_workload_sweep
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

run-forge-realistic-env-graceful-overload:
if: ${{ github.event_name != 'pull_request' && always() }}
Expand All @@ -172,6 +175,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
FORGE_TEST_SUITE: realistic_env_graceful_overload
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

run-forge-realistic-env-graceful-workload-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
Expand All @@ -184,6 +188,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 2100 # Run for 5 minutes per test, 7 tests.
FORGE_TEST_SUITE: realistic_env_graceful_workload_sweep
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

run-forge-realistic-env-fairness-workload-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
Expand All @@ -196,6 +201,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 900 # Run for 5 minutes per test, 3 tests.
FORGE_TEST_SUITE: realistic_env_fairness_workload_sweep
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

run-forge-realistic-network-tuned-for-throughput:
if: ${{ github.event_name != 'pull_request' && always() }}
Expand All @@ -209,6 +215,7 @@ jobs:
FORGE_TEST_SUITE: realistic_network_tuned_for_throughput
FORGE_ENABLE_PERFORMANCE: true
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

### Forge Correctness/Componenet/Stress tests

Expand All @@ -223,6 +230,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 2400 # Run for 40 minutes
FORGE_TEST_SUITE: consensus_stress_test
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

run-forge-workload-mix-test:
if: ${{ github.event_name != 'pull_request' && always() }}
Expand All @@ -235,6 +243,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: workload_mix
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

run-forge-single-vfn-perf:
if: ${{ github.event_name != 'pull_request' && always() }}
Expand All @@ -247,6 +256,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 480 # Run for 8 minutes
FORGE_TEST_SUITE: single_vfn_perf
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

run-forge-fullnode-reboot-stress-test:
if: ${{ github.event_name != 'pull_request' && always() }}
Expand All @@ -259,6 +269,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 1800 # Run for 30 minutes
FORGE_TEST_SUITE: fullnode_reboot_stress_test
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

### Compatibility Forge tests

Expand All @@ -275,6 +286,7 @@ jobs:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG_FOR_COMPAT_TEST }}
GIT_SHA: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }} # this is the git ref to checkout
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true

### Changing working quorum Forge tests

Expand All @@ -289,6 +301,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
FORGE_TEST_SUITE: changing_working_quorum_test
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true
FORGE_ENABLE_FAILPOINTS: true

run-forge-changing-working-quorum-test-high-load:
Expand All @@ -302,6 +315,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: changing_working_quorum_test_high_load
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true
FORGE_ENABLE_FAILPOINTS: true

# Measures PFN latencies with a constant TPS (with a realistic environment)
Expand All @@ -316,6 +330,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: pfn_const_tps_with_realistic_env
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true


# longest test for last, to get useful signal from short tests first
Expand All @@ -331,3 +346,4 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
FORGE_TEST_SUITE: realistic_env_max_load_large
POST_TO_SLACK: true
SEND_RESULTS_TO_TRUNK: true
16 changes: 16 additions & 0 deletions .github/workflows/workflow-run-forge.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ on:
required: false
type: string
description: The deployer profile used to spin up and configure forge infrastructure
SEND_RESULTS_TO_TRUNK:
required: false
type: boolean
description: Send forge results to trunk.io

env:
AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
Expand Down Expand Up @@ -118,6 +122,7 @@ env:
VERBOSE: true
FORGE_NUM_VALIDATORS: ${{ inputs.FORGE_NUM_VALIDATORS }}
FORGE_NUM_VALIDATOR_FULLNODES: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}
FORGE_JUNIT_XML_PATH: ${{ inputs.SEND_RESULTS_TO_TRUNK && '/tmp/test.xml' || '' }}

# TODO: should we migrate this to a composite action, so that we can skip it
# at the call site, and don't need to wrap each step in an if statement?
Expand Down Expand Up @@ -228,3 +233,14 @@ jobs:
# Print out whether the job was skipped.
- run: echo "Skipping forge test!"
if: ${{ inputs.SKIP_JOB }}

- name: Upload results
# Run this step even if the test step ahead fails
if: ${{ !inputs.SKIP_JOB && inputs.SEND_RESULTS_TO_TRUNK && !cancelled() }}
uses: trunk-io/analytics-uploader@main
with:
# Configured in the nextest.toml file
junit-paths: ${{ env.FORGE_JUNIT_XML_PATH }}
org-slug: aptoslabs
token: ${{ secrets.TRUNK_API_TOKEN }}
continue-on-error: true
Loading
Loading