From 06abbab8c2c68409bd8db39199c664e77243fc84 Mon Sep 17 00:00:00 2001 From: Perry Randall Date: Tue, 13 Sep 2022 14:38:54 -0700 Subject: [PATCH] [forge] Implement ability to run multiple tests --- .github/workflows/forge-continuous.yaml | 32 + .github/workflows/run-forge.yaml | 1 - .../forge-test-runner-template.fixture | 6 +- testsuite/fixtures/testFormatComment.fixture | 6 +- .../fixtures/testFormatPreComment.fixture | 6 +- .../testGetHumioForgeLinkAbsolute.fixture | 1 + .../testGetHumioForgeLinkRelative.fixture | 1 + .../testGetHumioLogsLinkAbsolute.fixture | 1 + .../testGetHumioLogsLinkRelative.fixture | 1 + testsuite/fixtures/testMain.fixture | 14 +- testsuite/fixtures/testMainComment.fixture | 4 +- testsuite/fixtures/testMainPreComment.fixture | 4 +- testsuite/forge.py | 859 ++++++++++++++++-- testsuite/forge_test.py | 136 ++- 14 files changed, 925 insertions(+), 147 deletions(-) create mode 100644 .github/workflows/forge-continuous.yaml create mode 100644 testsuite/fixtures/testGetHumioForgeLinkAbsolute.fixture create mode 100644 testsuite/fixtures/testGetHumioForgeLinkRelative.fixture create mode 100644 testsuite/fixtures/testGetHumioLogsLinkAbsolute.fixture create mode 100644 testsuite/fixtures/testGetHumioLogsLinkRelative.fixture diff --git a/.github/workflows/forge-continuous.yaml b/.github/workflows/forge-continuous.yaml new file mode 100644 index 0000000000000..43b1e9ab6d365 --- /dev/null +++ b/.github/workflows/forge-continuous.yaml @@ -0,0 +1,32 @@ +name: "Forge Continuous" +on: + workflow_dispatch: + schedule: + - cron: "0 * * * *" + pull_request: + paths: + - 'testsuite/**.py' + +# cancel redundant builds +concurrency: + # cancel redundant builds on PRs (only on PR, not on branches) + group: ${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.ref) || github.sha }} + cancel-in-progress: true + +permissions: + contents: read + id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry + issues: write + pull-requests: write + +jobs: + forge-continuous: + uses: ./.github/workflows/run-forge.yaml + secrets: inherit + with: + COMMENT_HEADER: forge-continuous + # Use the cache ID as the Forge namespace so we can limit Forge test concurrency on k8s, since Forge + # test lifecycle is separate from that of GHA. This protects us from the case where many Forge tests are triggered + # by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace. + FORGE_NAMESPACE: forge-continuous + FORGE_TEST_SUITE: continuous \ No newline at end of file diff --git a/.github/workflows/run-forge.yaml b/.github/workflows/run-forge.yaml index a756ca7116d32..0cce7a4e56466 100644 --- a/.github/workflows/run-forge.yaml +++ b/.github/workflows/run-forge.yaml @@ -28,7 +28,6 @@ on: FORGE_TEST_SUITE: required: false type: string - default: land_blocking description: Test suite to run POST_TO_SLACK: required: false diff --git a/testsuite/fixtures/forge-test-runner-template.fixture b/testsuite/fixtures/forge-test-runner-template.fixture index e1e8d8ffd5a7b..579156ad6d32a 100644 --- a/testsuite/fixtures/forge-test-runner-template.fixture +++ b/testsuite/fixtures/forge-test-runner-template.fixture @@ -1,10 +1,10 @@ apiVersion: v1 kind: Pod metadata: - name: potato-1659078000-asdf + name: forge-potato-1659078000-asdf labels: app.kubernetes.io/name: forge - forge-namespace: potato + forge-namespace: forge-potato forge-image-tag: forge_asdf spec: restartPolicy: Never @@ -18,7 +18,7 @@ spec: - -c - | ulimit -n 1048576 - forge --suite banana --duration-secs 123 --num-validators 10 --num-validator-fullnodes 20 --forge-cli-arg test k8s-swarm --image-tag asdf --upgrade-image-tag upgrade_asdf --namespace potato --test-arg + forge --suite banana --duration-secs 123 --num-validators 10 --num-validator-fullnodes 20 --forge-cli-arg test k8s-swarm --image-tag asdf --upgrade-image-tag upgrade_asdf --namespace forge-potato --test-arg env: - name: FORGE_TRIGGERED_BY diff --git a/testsuite/fixtures/testFormatComment.fixture b/testsuite/fixtures/testFormatComment.fixture index 6790f42c9524a..72aae0cb37f3f 100644 --- a/testsuite/fixtures/testFormatComment.fixture +++ b/testsuite/fixtures/testFormatComment.fixture @@ -3,8 +3,8 @@ performance benchmark with full nodes : 6795 TPS, 4370 ms latency, 6000 ms p99 latency,no expired txns Test Ok ``` -* [Grafana dashboard](https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&var-Datasource=Remote%20Prometheus%20Intern&var-namespace=potato&var-chain_name=matonet&from=1659078000000&to=1659078000000) -* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29%20%7C%20potato%20&live=false&start=1659078000000&end=1659078000000&widgetType=list-view&columns=%5B%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22%40timestamp%22%2C%22format%22%3A%22timestamp%22%2C%22width%22%3A180%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22level%22%2C%22format%22%3A%22text%22%2C%22width%22%3A54%7D%2C%7B%22type%22%3A%22link%22%2C%22openInNewBrowserTab%22%3A***%2C%22style%22%3A%22button%22%2C%22hrefTemplate%22%3A%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22textTemplate%22%3A%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22header%22%3A%22Forge%20PR%22%2C%22width%22%3A79%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.namespace%22%2C%22format%22%3A%22text%22%2C%22width%22%3A104%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.pod_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A126%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.container_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A85%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22message%22%2C%22format%22%3A%22text%22%7D%5D&newestAtBottom=***&showOnlyFirstLine=false) -* [(Deprecated) OpenSearch Logs](https://es.intern.aptosdev.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:'2022-07-29T07:00:00.000Z',to:'2022-07-29T07:00:00.000Z'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:chain_name,negate:!f,params:(query:matonet),type:phrase),query:(match_phrase:(chain_name:matonet))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:namespace,negate:!f,params:(query:potato),type:phrase),query:(match_phrase:(namespace:potato))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:hostname,negate:!f,params:(query:aptos-node-0-validator-0),type:phrase),query:(match_phrase:(hostname:aptos-node-0-validator-0)))),index:'90037930-aafc-11ec-acce-2d961187411f',interval:auto,query:(language:kuery,query:''),sort:!())) +* [Grafana dashboard](https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&var-Datasource=Remote%20Prometheus%20Intern&var-namespace=forge-potato&var-chain_name=matonet&from=1659078000000&to=1659078000000) +* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-potato&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1659078000000&end=1659078000000) +* [(Deprecated) OpenSearch Logs](https://es.intern.aptosdev.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:'2022-07-29T07:00:00.000Z',to:'2022-07-29T07:00:00.000Z'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:chain_name,negate:!f,params:(query:matonet),type:phrase),query:(match_phrase:(chain_name:matonet))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:namespace,negate:!f,params:(query:forge-potato),type:phrase),query:(match_phrase:(namespace:forge-potato))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:hostname,negate:!f,params:(query:aptos-node-0-validator-0),type:phrase),query:(match_phrase:(hostname:aptos-node-0-validator-0)))),index:'90037930-aafc-11ec-acce-2d961187411f',interval:auto,query:(language:kuery,query:''),sort:!())) * [Test runner output](https://banana) * Test run is land-blocking \ No newline at end of file diff --git a/testsuite/fixtures/testFormatPreComment.fixture b/testsuite/fixtures/testFormatPreComment.fixture index 5d46b6fcd902b..b20fd0df96afc 100644 --- a/testsuite/fixtures/testFormatPreComment.fixture +++ b/testsuite/fixtures/testFormatPreComment.fixture @@ -1,6 +1,6 @@ ### Forge is running suite `banana` on `asdf` ==> `upgrade_asdf` -* [Grafana dashboard (auto-refresh)](https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&var-Datasource=Remote%20Prometheus%20Intern&var-namespace=potato&var-chain_name=matonet&refresh=10s&from=now-15m&to=now) -* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29%20%7C%20potato%20&live=true&start=30m&widgetType=list-view&columns=%5B%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22%40timestamp%22%2C%22format%22%3A%22timestamp%22%2C%22width%22%3A180%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22level%22%2C%22format%22%3A%22text%22%2C%22width%22%3A54%7D%2C%7B%22type%22%3A%22link%22%2C%22openInNewBrowserTab%22%3A***%2C%22style%22%3A%22button%22%2C%22hrefTemplate%22%3A%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22textTemplate%22%3A%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22header%22%3A%22Forge%20PR%22%2C%22width%22%3A79%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.namespace%22%2C%22format%22%3A%22text%22%2C%22width%22%3A104%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.pod_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A126%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.container_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A85%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22message%22%2C%22format%22%3A%22text%22%7D%5D&newestAtBottom=***&showOnlyFirstLine=false) -* [(Deprecated) OpenSearch Logs](https://es.intern.aptosdev.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!f,value:10000),time:(from:now-15m,to:now))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:chain_name,negate:!f,params:(query:matonet),type:phrase),query:(match_phrase:(chain_name:matonet))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:namespace,negate:!f,params:(query:potato),type:phrase),query:(match_phrase:(namespace:potato))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:hostname,negate:!f,params:(query:aptos-node-0-validator-0),type:phrase),query:(match_phrase:(hostname:aptos-node-0-validator-0)))),index:'90037930-aafc-11ec-acce-2d961187411f',interval:auto,query:(language:kuery,query:''),sort:!())) +* [Grafana dashboard (auto-refresh)](https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&var-Datasource=Remote%20Prometheus%20Intern&var-namespace=forge-potato&var-chain_name=matonet&refresh=10s&from=now-15m&to=now) +* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-potato&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m) +* [(Deprecated) OpenSearch Logs](https://es.intern.aptosdev.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!f,value:10000),time:(from:now-15m,to:now))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:chain_name,negate:!f,params:(query:matonet),type:phrase),query:(match_phrase:(chain_name:matonet))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:namespace,negate:!f,params:(query:forge-potato),type:phrase),query:(match_phrase:(namespace:forge-potato))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'90037930-aafc-11ec-acce-2d961187411f',key:hostname,negate:!f,params:(query:aptos-node-0-validator-0),type:phrase),query:(match_phrase:(hostname:aptos-node-0-validator-0)))),index:'90037930-aafc-11ec-acce-2d961187411f',interval:auto,query:(language:kuery,query:''),sort:!())) * [Test runner output](https://banana) * Test run is land-blocking \ No newline at end of file diff --git a/testsuite/fixtures/testGetHumioForgeLinkAbsolute.fixture b/testsuite/fixtures/testGetHumioForgeLinkAbsolute.fixture new file mode 100644 index 0000000000000..a2a9f29d99915 --- /dev/null +++ b/testsuite/fixtures/testGetHumioForgeLinkAbsolute.fixture @@ -0,0 +1 @@ +https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-pr-2986+%7C+%22k8s.labels.app.kubernetes.io%2Fname%22+%3D+forge&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+3760%7D%5D&newestAtBottom=true&showOnlyFirstLine=false&live=true&start=30m \ No newline at end of file diff --git a/testsuite/fixtures/testGetHumioForgeLinkRelative.fixture b/testsuite/fixtures/testGetHumioForgeLinkRelative.fixture new file mode 100644 index 0000000000000..b57a899827439 --- /dev/null +++ b/testsuite/fixtures/testGetHumioForgeLinkRelative.fixture @@ -0,0 +1 @@ +https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-pr-2985+%7C+%22k8s.labels.app.kubernetes.io%2Fname%22+%3D+forge&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+3760%7D%5D&newestAtBottom=true&showOnlyFirstLine=false&live=true&start=30m \ No newline at end of file diff --git a/testsuite/fixtures/testGetHumioLogsLinkAbsolute.fixture b/testsuite/fixtures/testGetHumioLogsLinkAbsolute.fixture new file mode 100644 index 0000000000000..19f85535b4669 --- /dev/null +++ b/testsuite/fixtures/testGetHumioLogsLinkAbsolute.fixture @@ -0,0 +1 @@ +https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-pr-2984&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1659078000000&end=1659078000000 \ No newline at end of file diff --git a/testsuite/fixtures/testGetHumioLogsLinkRelative.fixture b/testsuite/fixtures/testGetHumioLogsLinkRelative.fixture new file mode 100644 index 0000000000000..efc1d9ed95678 --- /dev/null +++ b/testsuite/fixtures/testGetHumioLogsLinkRelative.fixture @@ -0,0 +1 @@ +https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-pr-2983&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m \ No newline at end of file diff --git a/testsuite/fixtures/testMain.fixture b/testsuite/fixtures/testMain.fixture index 7df2e8f3044a8..ca7dd9d26e6c7 100644 --- a/testsuite/fixtures/testMain.fixture +++ b/testsuite/fixtures/testMain.fixture @@ -2,15 +2,15 @@ Using the following image tags: forge: banana swarm: banana swarm upgrade (if applicable): banana +Using cluster: forge-big-1 === Start temp-pre-comment === -### Forge is running suite `land_blocking` on `banana` +### Forge is running suite `banana-test` on `banana` * [Grafana dashboard (auto-refresh)](https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&var-Datasource=Remote%20Prometheus%20Devinfra&var-namespace=forge-perry-1659078000&var-chain_name=forge-big-1&refresh=10s&from=now-15m&to=now) -* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29%20%7C%20forge-perry-1659078000%20&live=true&start=30m&widgetType=list-view&columns=%5B%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22%40timestamp%22%2C%22format%22%3A%22timestamp%22%2C%22width%22%3A180%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22level%22%2C%22format%22%3A%22text%22%2C%22width%22%3A54%7D%2C%7B%22type%22%3A%22link%22%2C%22openInNewBrowserTab%22%3A***%2C%22style%22%3A%22button%22%2C%22hrefTemplate%22%3A%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22textTemplate%22%3A%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22header%22%3A%22Forge%20PR%22%2C%22width%22%3A79%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.namespace%22%2C%22format%22%3A%22text%22%2C%22width%22%3A104%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.pod_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A126%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.container_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A85%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22message%22%2C%22format%22%3A%22text%22%7D%5D&newestAtBottom=***&showOnlyFirstLine=false) +* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-perry-1659078000&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m) * [(Deprecated) OpenSearch Logs](https://es.devinfra.aptosdev.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!f,value:10000),time:(from:now-15m,to:now))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:chain_name,negate:!f,params:(query:forge-big-1),type:phrase),query:(match_phrase:(chain_name:forge-big-1))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:namespace,negate:!f,params:(query:forge-perry-1659078000),type:phrase),query:(match_phrase:(namespace:forge-perry-1659078000))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:hostname,negate:!f,params:(query:aptos-node-0-validator-0),type:phrase),query:(match_phrase:(hostname:aptos-node-0-validator-0)))),index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',interval:auto,query:(language:kuery,query:''),sort:!())) * [Test runner output](None/None/actions/runs/None) * Test run is land-blocking === End temp-pre-comment === -Using cluster: forge-big-1 === Start temp-report === Forge test runner terminated: Trailing Log Lines: @@ -19,7 +19,7 @@ Debugging output: === End temp-report === === Start temp-comment === -### :x: Forge suite `land_blocking` failure on `banana` +### :x: Forge suite `banana-test` failure on `banana` ``` Forge test runner terminated: Trailing Log Lines: @@ -28,13 +28,13 @@ Debugging output: ``` * [Grafana dashboard](https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&var-Datasource=Remote%20Prometheus%20Devinfra&var-namespace=forge-perry-1659078000&var-chain_name=forge-big-1&from=1659078000000&to=1659078000000) -* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29%20%7C%20forge-perry-1659078000%20&live=false&start=1659078000000&end=1659078000000&widgetType=list-view&columns=%5B%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22%40timestamp%22%2C%22format%22%3A%22timestamp%22%2C%22width%22%3A180%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22level%22%2C%22format%22%3A%22text%22%2C%22width%22%3A54%7D%2C%7B%22type%22%3A%22link%22%2C%22openInNewBrowserTab%22%3A***%2C%22style%22%3A%22button%22%2C%22hrefTemplate%22%3A%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22textTemplate%22%3A%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22header%22%3A%22Forge%20PR%22%2C%22width%22%3A79%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.namespace%22%2C%22format%22%3A%22text%22%2C%22width%22%3A104%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.pod_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A126%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.container_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A85%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22message%22%2C%22format%22%3A%22text%22%7D%5D&newestAtBottom=***&showOnlyFirstLine=false) +* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-perry-1659078000&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1659078000000&end=1659078000000) * [(Deprecated) OpenSearch Logs](https://es.devinfra.aptosdev.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:'2022-07-29T07:00:00.000Z',to:'2022-07-29T07:00:00.000Z'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:chain_name,negate:!f,params:(query:forge-big-1),type:phrase),query:(match_phrase:(chain_name:forge-big-1))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:namespace,negate:!f,params:(query:forge-perry-1659078000),type:phrase),query:(match_phrase:(namespace:forge-perry-1659078000))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:hostname,negate:!f,params:(query:aptos-node-0-validator-0),type:phrase),query:(match_phrase:(hostname:aptos-node-0-validator-0)))),index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',interval:auto,query:(language:kuery,query:''),sort:!())) * [Test runner output](None/None/actions/runs/None) * Test run is land-blocking === End temp-comment === === Start temp-step-summary === -### :x: Forge suite `land_blocking` failure on `banana` +### :x: Forge suite `banana-test` failure on `banana` ``` Forge test runner terminated: Trailing Log Lines: @@ -43,7 +43,7 @@ Debugging output: ``` * [Grafana dashboard](https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&var-Datasource=Remote%20Prometheus%20Devinfra&var-namespace=forge-perry-1659078000&var-chain_name=forge-big-1&from=1659078000000&to=1659078000000) -* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29%20%7C%20forge-perry-1659078000%20&live=false&start=1659078000000&end=1659078000000&widgetType=list-view&columns=%5B%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22%40timestamp%22%2C%22format%22%3A%22timestamp%22%2C%22width%22%3A180%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22level%22%2C%22format%22%3A%22text%22%2C%22width%22%3A54%7D%2C%7B%22type%22%3A%22link%22%2C%22openInNewBrowserTab%22%3A***%2C%22style%22%3A%22button%22%2C%22hrefTemplate%22%3A%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22textTemplate%22%3A%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22header%22%3A%22Forge%20PR%22%2C%22width%22%3A79%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.namespace%22%2C%22format%22%3A%22text%22%2C%22width%22%3A104%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.pod_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A126%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.container_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A85%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22message%22%2C%22format%22%3A%22text%22%7D%5D&newestAtBottom=***&showOnlyFirstLine=false) +* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-perry-1659078000&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1659078000000&end=1659078000000) * [(Deprecated) OpenSearch Logs](https://es.devinfra.aptosdev.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:'2022-07-29T07:00:00.000Z',to:'2022-07-29T07:00:00.000Z'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:chain_name,negate:!f,params:(query:forge-big-1),type:phrase),query:(match_phrase:(chain_name:forge-big-1))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:namespace,negate:!f,params:(query:forge-perry-1659078000),type:phrase),query:(match_phrase:(namespace:forge-perry-1659078000))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:hostname,negate:!f,params:(query:aptos-node-0-validator-0),type:phrase),query:(match_phrase:(hostname:aptos-node-0-validator-0)))),index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',interval:auto,query:(language:kuery,query:''),sort:!())) * [Test runner output](None/None/actions/runs/None) * Test run is land-blocking diff --git a/testsuite/fixtures/testMainComment.fixture b/testsuite/fixtures/testMainComment.fixture index 1cd8a7a6ff088..247e17346b26d 100644 --- a/testsuite/fixtures/testMainComment.fixture +++ b/testsuite/fixtures/testMainComment.fixture @@ -1,4 +1,4 @@ -### :x: Forge suite `land_blocking` failure on `banana` +### :x: Forge suite `banana-test` failure on `banana` ``` Forge test runner terminated: Trailing Log Lines: @@ -7,7 +7,7 @@ Debugging output: ``` * [Grafana dashboard](https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&var-Datasource=Remote%20Prometheus%20Devinfra&var-namespace=forge-perry-1659078000&var-chain_name=forge-big-1&from=1659078000000&to=1659078000000) -* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29%20%7C%20forge-perry-1659078000%20&live=false&start=1659078000000&end=1659078000000&widgetType=list-view&columns=%5B%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22%40timestamp%22%2C%22format%22%3A%22timestamp%22%2C%22width%22%3A180%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22level%22%2C%22format%22%3A%22text%22%2C%22width%22%3A54%7D%2C%7B%22type%22%3A%22link%22%2C%22openInNewBrowserTab%22%3A***%2C%22style%22%3A%22button%22%2C%22hrefTemplate%22%3A%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22textTemplate%22%3A%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22header%22%3A%22Forge%20PR%22%2C%22width%22%3A79%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.namespace%22%2C%22format%22%3A%22text%22%2C%22width%22%3A104%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.pod_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A126%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.container_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A85%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22message%22%2C%22format%22%3A%22text%22%7D%5D&newestAtBottom=***&showOnlyFirstLine=false) +* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-perry-1659078000&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1659078000000&end=1659078000000) * [(Deprecated) OpenSearch Logs](https://es.devinfra.aptosdev.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:'2022-07-29T07:00:00.000Z',to:'2022-07-29T07:00:00.000Z'))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:chain_name,negate:!f,params:(query:forge-big-1),type:phrase),query:(match_phrase:(chain_name:forge-big-1))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:namespace,negate:!f,params:(query:forge-perry-1659078000),type:phrase),query:(match_phrase:(namespace:forge-perry-1659078000))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:hostname,negate:!f,params:(query:aptos-node-0-validator-0),type:phrase),query:(match_phrase:(hostname:aptos-node-0-validator-0)))),index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',interval:auto,query:(language:kuery,query:''),sort:!())) * [Test runner output](None/None/actions/runs/None) * Test run is land-blocking \ No newline at end of file diff --git a/testsuite/fixtures/testMainPreComment.fixture b/testsuite/fixtures/testMainPreComment.fixture index cc96bedc24a6a..68dc4a1252ad7 100644 --- a/testsuite/fixtures/testMainPreComment.fixture +++ b/testsuite/fixtures/testMainPreComment.fixture @@ -1,6 +1,6 @@ -### Forge is running suite `land_blocking` on `banana` +### Forge is running suite `banana-test` on `banana` * [Grafana dashboard (auto-refresh)](https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&var-Datasource=Remote%20Prometheus%20Devinfra&var-namespace=forge-perry-1659078000&var-chain_name=forge-big-1&refresh=10s&from=now-15m&to=now) -* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29%20%7C%20forge-perry-1659078000%20&live=true&start=30m&widgetType=list-view&columns=%5B%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22%40timestamp%22%2C%22format%22%3A%22timestamp%22%2C%22width%22%3A180%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22level%22%2C%22format%22%3A%22text%22%2C%22width%22%3A54%7D%2C%7B%22type%22%3A%22link%22%2C%22openInNewBrowserTab%22%3A***%2C%22style%22%3A%22button%22%2C%22hrefTemplate%22%3A%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22textTemplate%22%3A%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22header%22%3A%22Forge%20PR%22%2C%22width%22%3A79%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.namespace%22%2C%22format%22%3A%22text%22%2C%22width%22%3A104%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.pod_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A126%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.container_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A85%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%22message%22%2C%22format%22%3A%22text%22%7D%5D&newestAtBottom=***&showOnlyFirstLine=false) +* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C+forge-perry-1659078000&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m) * [(Deprecated) OpenSearch Logs](https://es.devinfra.aptosdev.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!f,value:10000),time:(from:now-15m,to:now))&_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:chain_name,negate:!f,params:(query:forge-big-1),type:phrase),query:(match_phrase:(chain_name:forge-big-1))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:namespace,negate:!f,params:(query:forge-perry-1659078000),type:phrase),query:(match_phrase:(namespace:forge-perry-1659078000))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',key:hostname,negate:!f,params:(query:aptos-node-0-validator-0),type:phrase),query:(match_phrase:(hostname:aptos-node-0-validator-0)))),index:'d0bc5e20-badc-11ec-9a50-89b84ac337af',interval:auto,query:(language:kuery,query:''),sort:!())) * [Test runner output](None/None/actions/runs/None) * Test run is land-blocking \ No newline at end of file diff --git a/testsuite/forge.py b/testsuite/forge.py index 68fa1adf4726a..8c8dd52831cf2 100644 --- a/testsuite/forge.py +++ b/testsuite/forge.py @@ -1,5 +1,4 @@ from __future__ import annotations - import multiprocessing @@ -31,15 +30,17 @@ from typing import ( Any, Callable, - ForwardRef, + Dict, Generator, List, Optional, Sequence, + Set, Tuple, TypedDict, Union, ) +from urllib.parse import ParseResult, urlunparse, urlencode @dataclass @@ -114,7 +115,7 @@ async def gen_run( output += chunk if stream_output: sys.stdout.write(chunk.decode("utf-8")) - await asyncio.sleep(0.1) + await asyncio.sleep(1) output += reader.read() exit_code = process.returncode assert exit_code is not None, "Process must have exited" @@ -216,29 +217,20 @@ def unlink(self, filename: str) -> None: "https://o11y.aptosdev.com/grafana/d/overview/overview?orgId=1&refresh=10s&" "var-Datasource=Remote%20Prometheus%20Devinfra" ) -HUMIO_LOGS_LINK = ( - "https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_insta" - "nce%3D%2A%29%20%7C%20$FILTER&widgetType=list-view&columns=%5B%7B%22type%22%3A%22field%2" - "2%2C%22fieldName%22%3A%22%40timestamp%22%2C%22format%22%3A%22timestamp%22%" - "2C%22width%22%3A180%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%" - "22level%22%2C%22format%22%3A%22text%22%2C%22width%22%3A54%7D%2C%7B%22type%" - "22%3A%22link%22%2C%22openInNewBrowserTab%22%3A***%2C%22style%22%3A%22butto" - "n%22%2C%22hrefTemplate%22%3A%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Fapt" - "os-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22tex" - "tTemplate%22%3A%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C%22he" - "ader%22%3A%22Forge%20PR%22%2C%22width%22%3A79%7D%2C%7B%22type%22%3A%22fiel" - "d%22%2C%22fieldName%22%3A%22k8s.namespace%22%2C%22format%22%3A%22text%22%2" - "C%22width%22%3A104%7D%2C%7B%22type%22%3A%22field%22%2C%22fieldName%22%3A%2" - "2k8s.pod_name%22%2C%22format%22%3A%22text%22%2C%22width%22%3A126%7D%2C%7B%" - "22type%22%3A%22field%22%2C%22fieldName%22%3A%22k8s.container_name%22%2C%22" - "format%22%3A%22text%22%2C%22width%22%3A85%7D%2C%7B%22type%22%3A%22field%22" - "%2C%22fieldName%22%3A%22message%22%2C%22format%22%3A%22text%22%7D%5D&newes" - "tAtBottom=***&showOnlyFirstLine=false" -) -def prometheus_port_forward() -> None: - os.execvp("kubectl", ["kubectl", "port-forward", "svc/aptos-node-mon-aptos-monitoring-prometheus", "9090"]) +def prometheus_port_forward(kubeconf: str) -> None: + os.execvp( + "kubectl", + [ + "kubectl", + "--kubeconfig", + kubeconf, + "port-forward", + "svc/aptos-node-mon-aptos-monitoring-prometheus", + "9090" + ] + ) class Process: @@ -368,13 +360,22 @@ def with_context( try: yield result result.set_debugging_output( - dump_forge_state(context.shell, context.forge_namespace) + dump_forge_state( + context.shell, + context.forge_namespace, + context.forge_cluster.kubeconf, + ) ) except Exception as e: result.set_state(ForgeState.FAIL) result.set_debugging_output( "{}\n{}\n".format( - str(e), dump_forge_state(context.shell, context.forge_namespace) + str(e), + dump_forge_state( + context.shell, + context.forge_namespace, + context.forge_cluster.kubeconf, + ) ) ) result._end_time = context.time.now() @@ -445,7 +446,7 @@ class ForgeContext: forge_image_tag: str image_tag: str upgrade_image_tag: str - forge_cluster_name: str + forge_cluster: ForgeCluster forge_test_suite: str forge_blocking: bool @@ -466,7 +467,7 @@ def report( @property def forge_chain_name(self) -> str: - forge_chain_name = self.forge_cluster_name.lstrip("aptos-") + forge_chain_name = self.forge_cluster.name.lstrip("aptos-") if "forge" not in forge_chain_name: forge_chain_name += "net" return forge_chain_name @@ -635,25 +636,183 @@ def get_dashboard_link( ) -def get_humio_logs_link( - forge_namespace: str, +def shorten_link(link: str) -> str: + headers = { + "x-api-key": os.getenv("SHORTENER_API_KEY"), + "Content-Type": "application/json" + } + body = json.dumps({ + "longUrl": link, + }) + try: + import requests + response = requests.post( + 'https://api.aws3.link/shorten', + headers=headers, + data=body + ) + return f"https://{response.json()['shortUrl']}" + # Dont fail if we fail to shorten + except Exception: + return link + + +def milliseconds(timestamp: datetime) -> int: + return int(timestamp.timestamp()) * 1000 + + +def apply_humio_time_filter( + urlparts: Dict[str, Union[str, bool, int]], time_filter: Union[bool, Tuple[datetime, datetime]], -) -> str: - filters = f"{forge_namespace}%20" +) -> Dict: if time_filter is True: - filters += "&live=true&start=30m" + urlparts = { + **urlparts, + "live": "true", + "start": "30m", + } elif isinstance(time_filter, tuple): - start_ms = int(time_filter[0].timestamp()) * 1000 - end_ms = int(time_filter[1].timestamp()) * 1000 - filters += f"&live=false&start={start_ms}&end={end_ms}" + start_ms = milliseconds(time_filter[0]) + end_ms = milliseconds(time_filter[1]) + urlparts = { + **urlparts, + "live": "false", + "start": start_ms, + "end": end_ms, + } else: raise Exception(f"Invalid refresh argument: {time_filter}") - return ( - HUMIO_LOGS_LINK - .replace("$FILTER", filters) + return urlparts + + +def get_humio_forge_link( + forge_namespace: str, + time_filter: Union[bool, Tuple[datetime, datetime]], +) -> str: + columns = [ + { + 'type': 'field', + 'fieldName': '@timestamp', + 'format': 'timestamp', + 'width': 180 + }, + { + "type": "link", + "openInNewBrowserTab": "***", + "style": "button", + "hrefTemplate": "https://github.com/aptos-labs/aptos-core/pull/{{fields[\"github_pr\"]}}", + "textTemplate": "{{fields[\"github_pr\"]}}", + "header": "Forge PR", + "width": 79 + }, + { + "type": "field", + "fieldName": "k8s.namespace", + "format": "text", + "width": 104 + }, + { + 'type': 'field', + 'fieldName': 'message', + 'format': 'text', + 'width': 3760 + }, + ] + urlparts = { + 'query': ( + '$forgeLogs(validator_instance=*)' + f' | {forge_namespace}' + ' | "k8s.labels.app.kubernetes.io/name" = forge' + ), + 'widgetType': 'list-view', + 'columns': json.dumps(columns), + 'newestAtBottom': 'true', + 'showOnlyFirstLine': 'false', + } + urlparts = apply_humio_time_filter(urlparts, time_filter) + query = urlencode(urlparts) + return urlunparse( + ParseResult( + 'https', + 'cloud.us.humio.com', + '/k8s/search', + '', + query, + '' + ) ) +def get_humio_logs_link( + forge_namespace: str, + time_filter: Union[bool, Tuple[datetime, datetime]], +) -> str: + query = f'$forgeLogs(validator_instance=*) | {forge_namespace}' + columns = [ + { + "type": "field", + "fieldName": "@timestamp", + "format": "timestamp", + "width": 180 + }, + { + "type": "field", + "fieldName": "level", + "format": "text", + "width": 54 + }, + { + "type": "link", + "openInNewBrowserTab": "***", + "style": "button", + "hrefTemplate": "https://github.com/aptos-labs/aptos-core/pull/{{fields[\"github_pr\"]}}", + "textTemplate": "{{fields[\"github_pr\"]}}", + "header": "Forge PR", + "width": 79 + }, + { + "type": "field", + "fieldName": "k8s.namespace", + "format": "text", + "width": 104 + }, + { + "type": "field", + "fieldName": "k8s.pod_name", + "format": "text", + "width": 126 + }, + { + "type": "field", + "fieldName": "k8s.container_name", + "format": "text", + "width": 85 + }, + { + "type": "field", + "fieldName": "message", + "format": "text" + }, + ] + urlparts = { + 'query': query, + 'widgetType': 'list-view', + 'columns': json.dumps(columns), + 'newestAtBottom': '***', + 'showOnlyFirstLine': 'false', + } + urlparts = apply_humio_time_filter(urlparts, time_filter) + return urlunparse( + ParseResult( + 'https', + 'cloud.us.humio.com', + '/k8s/search', + '', + urlencode(urlparts), + '' + ) + ) + def format_github_info(context: ForgeContext) -> str: if not context.github_job_url: return "" @@ -680,7 +839,7 @@ def get_testsuite_images(context: ForgeContext) -> str: def format_pre_comment(context: ForgeContext) -> str: dashboard_link = get_dashboard_link( - context.forge_cluster_name, + context.forge_cluster.name, context.forge_namespace, context.forge_chain_name, True, @@ -708,7 +867,7 @@ def format_pre_comment(context: ForgeContext) -> str: def format_comment(context: ForgeContext, result: ForgeResult) -> str: dashboard_link = get_dashboard_link( - context.forge_cluster_name, + context.forge_cluster.name, context.forge_namespace, context.forge_chain_name, (result.start_time, result.end_time), @@ -763,12 +922,18 @@ def run(self, context: ForgeContext) -> ForgeResult: raise NotImplementedError -def dump_forge_state(shell: Shell, forge_namespace: str) -> str: +def dump_forge_state( + shell: Shell, + forge_namespace: str, + kubeconf: str, +) -> str: try: output = ( shell.run( [ "kubectl", + "--kubeconfig", + kubeconf, "get", "pods", "-n", @@ -783,10 +948,16 @@ def dump_forge_state(shell: Shell, forge_namespace: str) -> str: return f"Failed to get debugging output: {e}" -def find_the_killer(shell: Shell, forge_namespace) -> str: +def find_the_killer( + shell: Shell, + forge_namespace: str, + kubeconf: str, +) -> str: killer = shell.run( [ "kubectl", + "--kubeconfig", + kubeconf, "get", "pod", "-l", @@ -804,7 +975,7 @@ def run(self, context: ForgeContext) -> ForgeResult: context.filesystem.rlimit( resource.RLIMIT_NOFILE, resource.RLIM_INFINITY, resource.RLIM_INFINITY ) - port_forward_process = context.processes.spawn(prometheus_port_forward) + port_forward_process = context.processes.spawn(lambda: prometheus_port_forward(context.forge_cluster.kubeconf)) with ForgeResult.with_context(context) as forge_result: result = context.shell.run( @@ -839,6 +1010,8 @@ def run(self, context: ForgeContext) -> ForgeResult: context.shell.run( [ "kubectl", + "--kubeconfig", + context.forge_cluster.kubeconf, "delete", "pod", "-n", @@ -851,6 +1024,8 @@ def run(self, context: ForgeContext) -> ForgeResult: context.shell.run( [ "kubectl", + "--kubeconfig", + context.forge_cluster.kubeconf, "wait", "-n", "default", @@ -881,11 +1056,19 @@ def run(self, context: ForgeContext) -> ForgeResult: specfile = context.filesystem.mkstemp() context.filesystem.write(specfile, rendered.encode()) context.shell.run( - ["kubectl", "apply", "-n", "default", "-f", specfile] + [ + "kubectl", + "--kubeconfig", + context.forge_cluster.kubeconf, + "apply", + "-n", "default", + "-f", specfile] ).unwrap() context.shell.run( [ "kubectl", + "--kubeconfig", + context.forge_cluster.kubeconf, "wait", "-n", "default", @@ -899,7 +1082,14 @@ def run(self, context: ForgeContext) -> ForgeResult: streaming = True while state is None: forge_logs = context.shell.run( - ["kubectl", "logs", "-n", "default", "-f", forge_pod_name], + [ + "kubectl", + "--kubeconfig", + context.forge_cluster.kubeconf, + "logs", + "-n", "default", + "-f", forge_pod_name + ], stream_output=streaming, ) @@ -914,6 +1104,8 @@ def run(self, context: ForgeContext) -> ForgeResult: context.shell.run( [ "kubectl", + "--kubeconfig", + context.forge_cluster.kubeconf, "get", "pod", "-n", @@ -934,7 +1126,11 @@ def run(self, context: ForgeContext) -> ForgeResult: elif re.findall(r"not\s*found", forge_status, re.IGNORECASE): state = ForgeState.SKIP forge_result.set_debugging_output( - find_the_killer(context.shell, context.forge_namespace) + find_the_killer( + context.shell, + context.forge_namespace, + context.forge_cluster.kubeconf, + ) ) else: state = ForgeState.FAIL @@ -980,12 +1176,6 @@ def list_eks_clusters(shell: Shell) -> List[str]: raise AwsError("Failed to list eks clusters") from e -def set_current_cluster(shell: Shell, forge_cluster_name: str) -> None: - shell.run( - ["aws", "eks", "update-kubeconfig", "--name", forge_cluster_name] - ).unwrap() - - async def write_cluster_config( shell: Shell, forge_cluster_name: str, temp: str ) -> None: @@ -1113,7 +1303,9 @@ def image_exists(shell: Shell, image_name: str, image_tag: str) -> bool: def sanitize_forge_resource_name(forge_resource: str) -> str: - """Sanitize the intended forge resource name to be a valid k8s resource name""" + """ + Sanitize the intended forge resource name to be a valid k8s resource name + """ max_length = 64 sanitized_namespace = "" for i, c in enumerate(forge_resource): @@ -1123,6 +1315,8 @@ def sanitize_forge_resource_name(forge_resource: str) -> str: sanitized_namespace += c else: sanitized_namespace += "-" + if not forge_resource.startswith("forge-"): + raise Exception("Forge resource name must start with 'forge-'") return sanitized_namespace @@ -1208,6 +1402,73 @@ def create_forge_command( return forge_args +async def run_multiple( + context: SystemContext, + forge_test_suites: List[str], + disabled_suites: Set[str], + forge_namespace: str, + forge_pre_comment: Optional[str], + forge_comment: Optional[str], + forge_runner_mode: Optional[str], +) -> None: + # Remove formatting environment variables + os.environ["FORGE_OUTPUT"] = "" + os.environ["FORGE_REPORT"] = "" + os.environ["FORGE_PRE_COMMENT"] = "" + os.environ["FORGE_COMMENT"] = "" + + pending_results = [] + pending_suites = [] + pending_comment = [] + + for suite in forge_test_suites: + new_namespace = f"{forge_namespace}-{suite}" + short_link = shorten_link(get_humio_forge_link(new_namespace, True)) + pending_comment.append(f"Running {suite}: {short_link}") + if forge_runner_mode != "pre-forge": + pending_results.append( + context.shell.gen_run( + [ + # TODO figure out which other args we should forward + # This might only work from github for starters + sys.executable, __file__, + "test", + "--forge-test-suite", suite, + "--forge-namespace", new_namespace, + ], + ) + ) + pending_suites.append((suite, new_namespace)) + print("\n".join(pending_comment)) + if forge_runner_mode == "pre-forge": + if forge_pre_comment: + context.filesystem.write( + forge_pre_comment, + "\n".join(pending_comment).encode(), + ) + else: + final_forge_comment = [] + results = await asyncio.gather(*pending_results) + assert len(results) == len(pending_suites) + failed = False + for i, result in enumerate(results): + suite, namespace = pending_suites[i] + if result.succeeded(): + final_forge_comment.append(f"{suite} succeeded") + else: + failed = suite not in disabled_suites + disabled = " (disabled)" if suite in disabled_suites else "" + final_forge_comment.append(f"{suite} failed{disabled}") + final_forge_comment.append( + f"Run {'failed' if failed else 'succeeded'}" + ) + if forge_comment: + context.filesystem.write( + forge_comment, + "\n".join(final_forge_comment).encode() + ) + + @main.command() # output files @envoption("FORGE_OUTPUT") @@ -1227,7 +1488,7 @@ def create_forge_command( @envoption("FORGE_ENABLE_HAPROXY") @envoption("FORGE_ENABLE_FAILPOINTS") @envoption("FORGE_ENABLE_PERFORMANCE") -@envoption("FORGE_TEST_SUITE", "land_blocking") +@envoption("FORGE_TEST_SUITE") @envoption("FORGE_RUNNER_DURATION_SECS", "300") @envoption("FORGE_IMAGE_TAG") @envoption("IMAGE_TAG") @@ -1241,9 +1502,22 @@ def create_forge_command( @envoption("GITHUB_REPOSITORY") @envoption("GITHUB_RUN_ID") @envoption("GITHUB_STEP_SUMMARY") -@click.option("--cargo-args", multiple=True, help="Cargo args to pass to forge local runner") -@click.option("--forge-cli-args", multiple=True, help="Forge cli args to pass to forge cli") -@click.option("--test-args", multiple=True, help="Test args to pass to forge test subcommand") +@click.option( + "--cargo-args", + multiple=True, + help="Cargo args to pass to forge local runner", +) +@click.option( + "--forge-cli-args", + multiple=True, + help="Forge cli args to pass to forge cli" +) +@click.option( + "--test-args", + multiple=True, + help="Test args to pass to forge test subcommand" +) +@click.argument("test_suites", nargs=-1) def test( forge_output: Optional[str], forge_report: Optional[str], @@ -1276,6 +1550,7 @@ def test( cargo_args: Optional[List[str]], forge_cli_args: Optional[List[str]], test_args: Optional[List[str]], + test_suites: Tuple[str], ) -> None: """Run a forge test""" shell = LocalShell(verbose == "true") @@ -1287,6 +1562,58 @@ def test( config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) config.init() + if forge_namespace is None: + forge_namespace = f"forge-{processes.user()}-{time.epoch()}" + + assert forge_namespace is not None, "Forge namespace is required" + + forge_namespace = sanitize_forge_resource_name(forge_namespace) + + all_suites = list(test_suites) + if forge_test_suite: + all_suites.append(forge_test_suite) + if not all_suites: + forge_test_suite = "default" + + # Resolve suites to tests + all_resolved_suites = [] + enabled_resolved_suites = [] + for suite in all_suites: + config_suites = config.get("test_suites") + if suite in config_suites: + enabled_resolved_suites.extend( + config_suites[suite]["enabled_tests"].keys() + ) + all_resolved_suites.extend( + config_suites[suite]["all_tests"].keys() + ) + else: + enabled_resolved_suites.append(suite) + all_resolved_suites.append(suite) + + disabled_resolved_suites = ( + set(all_resolved_suites) - set(enabled_resolved_suites) + ) + + if len(all_resolved_suites) == 0: + print("No tests to run") + return + elif len(all_resolved_suites) == 1: + forge_test_suite = enabled_resolved_suites[0] + else: + asyncio.run( + run_multiple( + context, + all_resolved_suites, + disabled_resolved_suites, + forge_namespace, + forge_pre_comment, + forge_comment, + forge_runner_mode, + ) + ) + return + aws_account_num = None try: aws_account_num = get_aws_account_num(shell) @@ -1294,25 +1621,12 @@ def test( print(f"Warning: failed to get AWS account number: {e}") # Perform cluster selection - current_cluster = None - try: - current_cluster = get_current_cluster_name(shell) - except Exception as e: - print(f"Warning: failed to get current cluster name: {e}") - if not forge_cluster_name or balance_clusters: cluster_names = config.get("enabled_clusters") forge_cluster_name = random.choice(cluster_names) assert forge_cluster_name, "Forge cluster name is required" - if forge_namespace is None: - forge_namespace = f"forge-{processes.user()}-{time.epoch()}" - - forge_namespace = sanitize_forge_resource_name(forge_namespace) - - assert forge_namespace is not None, "Forge namespace is required" - # These features and profile flags are set as strings enable_failpoints_feature = forge_enable_failpoints == "true" enable_performance_profile = forge_enable_performance == "true" @@ -1335,7 +1649,8 @@ def test( enable_performance_profile=enable_performance_profile, ) ) - # This might not work as intended because we dont know if that revision passed forge + # This might not work as intended because we dont know if that revision + # passed forge image_tag = image_tag or second_latest_image forge_image_tag = forge_image_tag or default_latest_image upgrade_image_tag = upgrade_image_tag or default_latest_image @@ -1387,8 +1702,13 @@ def test( forge_cli_args=forge_cli_args, test_args=test_args, ) + + print(f"Using cluster: {forge_cluster_name}") + temp = context.filesystem.mkstemp() + forge_cluster = ForgeCluster(forge_cluster_name, temp) + asyncio.run(forge_cluster.write(context.shell)) - context = ForgeContext( + forge_context = ForgeContext( shell=shell, filesystem=filesystem, processes=processes, @@ -1400,7 +1720,7 @@ def test( upgrade_image_tag=upgrade_image_tag, forge_namespace=forge_namespace, keep_port_forwards=forge_namespace_keep == "true", - forge_cluster_name=forge_cluster_name, + forge_cluster=forge_cluster, forge_test_suite=forge_test_suite, forge_blocking=forge_blocking == "true", github_actions=github_actions, @@ -1413,9 +1733,9 @@ def test( } # Maybe this should be its own command? - pre_comment = format_pre_comment(context) + pre_comment = format_pre_comment(forge_context) if forge_pre_comment: - context.report( + forge_context.report( ForgeResult.empty(), [ForgeFormatter(forge_pre_comment, lambda *_: pre_comment)], ) @@ -1426,41 +1746,41 @@ def test( return try: - print(f"Using cluster: {forge_cluster_name}") - set_current_cluster(shell, forge_cluster_name) - forge_runner = forge_runner_mapping[forge_runner_mode]() - result = forge_runner.run(context) + result = forge_runner.run(forge_context) outputs = [] if forge_output: - outputs.append(ForgeFormatter(forge_output, lambda *_: result.output)) + outputs.append( + ForgeFormatter(forge_output, lambda *_: result.output) + ) if forge_report: outputs.append(ForgeFormatter(forge_report, format_report)) else: - print(format_report(context, result)) + print(format_report(forge_context, result)) if forge_comment: outputs.append(ForgeFormatter(forge_comment, format_comment)) else: - print(format_comment(context, result)) + print(format_comment(forge_context, result)) if github_step_summary: outputs.append(ForgeFormatter(github_step_summary, format_comment)) - context.report(result, outputs) + forge_context.report(result, outputs) - print(result.format(context)) + print(result.format(forge_context)) if not result.succeeded() and forge_blocking == "true": raise SystemExit(1) except Exception as e: - if current_cluster: - try: - set_current_cluster(shell, current_cluster) - except Exception as ee: - print(f"Warning: failed to restore current cluster: {ee}") - print("Set cluster manually with aws eks update-kubeconfig --name {current_cluster}") raise Exception( - "Forge state:\n" + dump_forge_state(shell, forge_namespace) + "\n".join([ + "Forge state:", + dump_forge_state( + shell, + forge_namespace, + forge_cluster.kubeconf, + ) + ]) ) from e @@ -1473,7 +1793,9 @@ class ForgeJob: @classmethod def from_pod(cls, cluster: ForgeCluster, pod: GetPodsItem) -> ForgeJob: return cls( - name=pod["metadata"]["name"], phase=pod["status"]["phase"], cluster=cluster + name=pod["metadata"]["name"], + phase=pod["status"]["phase"], + cluster=cluster, ) def running(self): @@ -1567,7 +1889,13 @@ def unlink_tempfiles(): return all_jobs -@main.command("list-jobs") +@main.group("job") +def job() -> None: + """Subcommands for managing forge jobs""" + pass + + +@job.command("list") @click.option("--phase", multiple=True, help="Only show jobs in this phase") @click.option("--regex", help="Only show jobs matching this regex") def list_jobs( @@ -1647,11 +1975,22 @@ def tail( DEFAULT_CONFIG_KEY = "forge-wrapper-config.json" +class TestConfig(TypedDict): + name: str + + +class TestSuite(TypedDict): + name: str + all_tests: Dict[str, TestConfig] + enabled_tests: Dict[str, TestConfig] + + # All changes to this struct must be backwards compatible # i.e. its ok to add a new field, but not to remove one class ForgeConfigValue(TypedDict): enabled_clusters: List[str] all_clusters: List[str] + test_suites: Dict[str, TestSuite] def default_forge_config() -> ForgeConfigValue: @@ -1748,6 +2087,8 @@ def read(self) -> object: class ForgeConfig: + NONE_SENTINEL = object() + def __init__(self, backend: ForgeConfigBackend) -> None: self.backend = backend self.config: ForgeConfigValue = default_forge_config() @@ -1758,8 +2099,11 @@ def create(self) -> None: def init(self) -> None: self.config = ensure_forge_config(self.backend.read()) - def get(self, key: str) -> Any: - return self.config[key] + def get(self, key: str, default: Optional[Any] = NONE_SENTINEL) -> Any: + value = self.config.get(key, default) + if value is self.NONE_SENTINEL: + raise Exception(f"Missing key {key}") + return value def set(self, key, value, validate: bool = True) -> None: new_config = {**self.config, key: value} @@ -1845,5 +2189,328 @@ def set_config( config.flush() +@config.command("edit") +@click.pass_context +def config_edit(ctx: click.Context) -> None: + shell = LocalShell(True) + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + config.init() + + temp = filesystem.mkstemp() + filesystem.write(temp, json.dumps(config.dump(), indent=4).encode()) + editor = os.getenv("EDITOR", "vim") + os.system(f"{editor} {temp}") + ctx.invoke(set_config, config_path=temp) + + +@config.group("cluster") +def cluster_config() -> None: + """Manage forge cluster configuration""" + pass + + +@cluster_config.command("delete") +@click.argument("cluster") +@click.option("--force", is_flag=True, help="Disable config validation") +def cluster_config_delete( + cluster: str, + force: Optional[bool], +) -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + enabled_clusters = config.get("enabled_clusters") + if cluster in enabled_clusters and not force: + raise Exception( + f"Cluster {cluster} is enabled, use --force to delete anyway" + ) + all_clusters = config.get("all_clusters") + all_clusters.remove(cluster) + config.set("all_clusters", all_clusters) + + config.flush() + + +@cluster_config.command("add") +@click.argument("cluster") +def cluster_config_add(cluster: str) -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + all_clusters = config.get("all_clusters") + if cluster in all_clusters: + raise Exception(f"Cluster {cluster} already exists") + all_clusters.append(cluster) + config.set("all_clusters", all_clusters) + + config.flush() + + +@cluster_config.command("enable") +@click.argument("cluster") +def cluster_config_enable(cluster: str) -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + enabled_clusters = config.get("enabled_clusters") + if cluster in enabled_clusters: + raise Exception( + f"Cluster {cluster} is already enabled" + ) + enabled_clusters.append(cluster) + config.set("enabled_clusters", enabled_clusters) + + config.flush() + + +@cluster_config.command("disable") +@click.argument("cluster") +def cluster_config_disable( + cluster: str, +) -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + enabled_clusters = config.get("enabled_clusters") + enabled_clusters.remove(cluster) + config.set("enabled_clusters", enabled_clusters) + + config.flush() + + +@cluster_config.command("list") +def cluster_config_list() -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + enabled_clusters = config.get("enabled_clusters") + for cluster in config.get("all_clusters"): + if cluster in enabled_clusters: + fg = "green" + enabled = " [enabled]" + else: + fg = "white" + enabled = "" + + click.secho(f"{cluster}{enabled}", fg=fg) + + config.flush() + + +@config.group("test") +def test_config() -> None: + """Manage forge test configuration""" + pass + + +@test_config.command("add") +@click.argument("suite_name") +@click.argument("test_name", required=False) +def test_config_add( + suite_name: str, + test_name: Optional[str], +) -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + suites = config.get("test_suites") + if suites is None: + raise Exception("Failed to get suites") + test_suite = suites.get( + suite_name, + { + "name": suite_name, + "all_tests": {}, + "enabled_tests": {}, + } + ) + + if test_name in test_suite["all_tests"]: + raise Exception(f"Test {test_name} already exists") + + if test_name: + test_suite["all_tests"][test_name]: TestConfig = { + "name": test_name, + } + + suites[suite_name] = test_suite + + config.set("test_suites", suites) + config.flush() + + +@test_config.command("show") +@click.argument("suite", required=False) +def test_config_show( + suite: Optional[str], +) -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + test_suites = config.get("test_suites") + for suite_name in test_suites: + print(suite_name) + if suite and suite_name != suite: + continue + suite_config = test_suites[suite_name] + for test_name in suite_config["all_tests"]: + if test_name in suite_config.get("enabled_tests"): + fg = "green" + enabled = " [enabled]" + else: + fg = "" + enabled = "" + + click.secho(f" - {test_name}{enabled}", fg=fg) + + + +@test_config.command("list") +def test_config_list() -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + test_suites = config.get("test_suites") + + for suite_name in test_suites: + print(suite_name) + + +@test_config.command("delete") +@click.argument("suite_name") +@click.argument("test_name", required=False) +def test_config_delete( + suite_name: str, + test_name: Optional[str], +) -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + suites = config.get("test_suites") + + if test_name: + suite_config = suites.get(suite_name) + if test_name in suite_config.get("enabled_tests"): + raise Exception(f"Cannot delete enabled test {test_name}") + del suite_config["enabled_tests"][test_name] + suites[suite_name] = suite_config + else: + del suites[suite_name] + + config.set("test_suites", suites) + config.flush() + + +@test_config.command("enable") +@click.argument("suite_name") +@click.argument("test_name") +def test_config_enable( + suite_name: str, + test_name: str, +) -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + suites = config.get("test_suites") + suite_config = suites.get(suite_name) + + if test_name in suite_config.get("enabled_tests"): + raise Exception(f"{test_name} is already enabled") + + test_config = suite_config.get("all_tests").get(test_name) + if test_config is None: + raise Exception(f"Cannot find test {test_name}") + + suite_config["enabled_tests"][test_name] = test_config + suites[suite_name] = suite_config + + config.set("test_suites", suites) + config.flush() + + +@test_config.command("disable") +@click.argument("suite_name") +@click.argument("test_name") +def test_config_disable( + suite_name: str, + test_name: str, +) -> None: + shell = LocalShell() + filesystem = LocalFilesystem() + processes = SystemProcesses() + context = SystemContext(shell, filesystem, processes) + config = ForgeConfig(S3ForgeConfigBackend(context, DEFAULT_CONFIG)) + + config.init() + + suites = config.get("test_suites") + suite_config = suites.get(suite_name) + + if test_name not in suite_config.get("enabled_tests"): + raise Exception(f"{test_name} is not enabled") + + test_config = suite_config.get("all_tests").get(test_name) + if test_config is None: + raise Exception(f"Cannot find test {test_name}") + + del suite_config["enabled_tests"][test_name] + suites[suite_name] = suite_config + + config.set("test_suites", suites) + config.flush() + if __name__ == "__main__": main() diff --git a/testsuite/forge_test.py b/testsuite/forge_test.py index bb43fa39e30fc..10b9f67d1b99a 100644 --- a/testsuite/forge_test.py +++ b/testsuite/forge_test.py @@ -24,6 +24,7 @@ from forge import ( Filesystem, ForgeCluster, + ForgeConfigBackend, ForgeContext, ForgeFormatter, ForgeJob, @@ -52,18 +53,18 @@ format_report, get_all_forge_jobs, get_dashboard_link, + get_humio_forge_link, get_humio_logs_link, get_testsuite_images, get_validator_logs_link, list_eks_clusters, main, sanitize_forge_resource_name, + validate_forge_config, ) from click.testing import CliRunner -from testsuite.forge import ForgeConfigBackend, ensure_forge_config, validate_forge_config - class HasAssertMultiLineEqual(Protocol): def assertMultiLineEqual(self, first: str, second: str, msg: Any = ...) -> None: @@ -294,7 +295,7 @@ def fake_context( forge_num_validator_fullnodes="20", image_tag="asdf", upgrade_image_tag="upgrade_asdf", - forge_namespace="potato", + forge_namespace="forge-potato", forge_namespace_reuse="false", forge_namespace_keep="false", forge_enable_haproxy="false", @@ -307,9 +308,9 @@ def fake_context( forge_image_tag="forge_asdf", image_tag="asdf", upgrade_image_tag="upgrade_asdf", - forge_namespace="potato", + forge_namespace="forge-potato", keep_port_forwards=False, - forge_cluster_name="tomato", + forge_cluster=ForgeCluster("tomato", "kubeconf"), forge_test_suite="banana", forge_blocking=True, github_actions="false", @@ -334,7 +335,7 @@ def testLocalRunner(self) -> None: "test", "k8s-swarm", "--image-tag", "asdf", "--upgrade-image-tag", "upgrade_asdf", - "--namespace", "potato", + "--namespace", "forge-potato", "--port-forward", "--test-arg" ]) @@ -342,7 +343,7 @@ def testLocalRunner(self) -> None: OrderedDict( [ (cargo_run, RunResult(0, b"orange"),), - ("kubectl get pods -n potato", RunResult(0, b"Pods")), + ("kubectl --kubeconfig kubeconf get pods -n forge-potato", RunResult(0, b"Pods")), ] ) ) @@ -361,27 +362,33 @@ def testK8sRunner(self) -> None: OrderedDict( [ ( - "kubectl delete pod -n default -l forge-namespace=potato --force", + "kubectl --kubeconfig kubeconf delete pod -n default -l forge-namespace=forge-potato --force", RunResult(0, b""), ), ( - "kubectl wait -n default --for=delete pod -l forge-namespace=potato", + "kubectl --kubeconfig kubeconf wait -n default --for=delete pod -l forge-namespace=forge-potato", RunResult(0, b""), ), - ("kubectl apply -n default -f temp1", RunResult(0, b"")), ( - "kubectl wait -n default --timeout=5m --for=condition=Ready pod/potato-1659078000-asdf", + "kubectl --kubeconfig kubeconf apply -n default -f temp1", RunResult(0, b""), ), ( - "kubectl logs -n default -f potato-1659078000-asdf", + "kubectl --kubeconfig kubeconf wait -n default --timeout=5m --for=condition=Ready pod/forge-potato-1659078000-asdf", RunResult(0, b""), ), ( - "kubectl get pod -n default potato-1659078000-asdf -o jsonpath='{.status.phase}'", + "kubectl --kubeconfig kubeconf logs -n default -f forge-potato-1659078000-asdf", + RunResult(0, b""), + ), + ( + "kubectl --kubeconfig kubeconf get pod -n default forge-potato-1659078000-asdf -o jsonpath='{.status.phase}'", RunResult(0, b"Succeeded"), ), - ("kubectl get pods -n potato", RunResult(0, b"Pods")), + ( + "kubectl --kubeconfig kubeconf get pods -n forge-potato", + RunResult(0, b"Pods"), + ), ] ) ) @@ -548,11 +555,27 @@ def testReport(self) -> None: filesystem.assert_reads(self) filesystem.assert_writes(self) - def testHumioLogLink(self) -> None: + def testGetHumioLogsLinkRelative(self) -> None: link = get_humio_logs_link("forge-pr-2983", True) - self.assertFixture(link, "testHumioLogLink.fixture") + self.assertFixture(link, "testGetHumioLogsLinkRelative.fixture") self.assertIn("forge-pr-2983", link) + def testGetHumioLogsLinkAbsolute(self) -> None: + time = FakeTime() + link = get_humio_logs_link("forge-pr-2984", (time.now(), time.now())) + self.assertFixture(link, "testGetHumioLogsLinkAbsolute.fixture") + self.assertIn("forge-pr-2984", link) + + def testGetHumioForgeLinkRelative(self) -> None: + link = get_humio_forge_link("forge-pr-2985", True) + self.assertFixture(link, "testGetHumioForgeLinkRelative.fixture") + self.assertIn("forge-pr-2985", link) + + def testGetHumioForgeLinkAbsolute(self) -> None: + link = get_humio_forge_link("forge-pr-2986", True) + self.assertFixture(link, "testGetHumioForgeLinkAbsolute.fixture") + self.assertIn("forge-pr-2986", link) + def testValidatorLogsLink(self) -> None: self.assertFixture( get_validator_logs_link("aptos-perry", "perrynet", True), @@ -614,14 +637,17 @@ def testFormatReport(self) -> None: ) def testSanitizeForgeNamespaceSlashes(self) -> None: - namespace_with_slash = "banana/apple" + namespace_with_slash = "forge-banana/apple" namespace = sanitize_forge_resource_name(namespace_with_slash) - self.assertEqual(namespace, "banana-apple") + self.assertEqual(namespace, "forge-banana-apple") def testSanitizeForgeNamespaceTooLong(self) -> None: - namespace_too_long = "a" * 10000 + namespace_too_long = "forge-" + "a" * 10000 namespace = sanitize_forge_resource_name(namespace_too_long) - self.assertEqual(namespace, "a" * 64) + self.assertEqual( + namespace, + "forge-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) class ForgeMainTests(unittest.TestCase, AssertFixtureMixin): @@ -631,44 +657,46 @@ def testMain(self) -> None: runner = CliRunner() shell = SpyShell(OrderedDict([ ('aws sts get-caller-identity', RunResult(0, b'{"Account": "123456789012"}')), - ('kubectl config current-context', RunResult(0, b'aptos-banana')), ('git rev-parse HEAD~0', RunResult(0, b'banana')), ( 'aws ecr describe-images --repository-name aptos/validator --im' 'age-ids imageTag=banana', RunResult(0, b''), ), - ('aws eks update-kubeconfig --name forge-big-1', RunResult(0, b'')), ( - 'kubectl delete pod -n default -l forge-namespace=forge-perry-1659078000 ' + 'aws eks update-kubeconfig --name forge-big-1 --kubeconfig temp1', + RunResult(0, b''), + ), + ( + 'kubectl --kubeconfig temp1 delete pod -n default -l forge-namespace=forge-perry-1659078000 ' '--force', RunResult(0, b''), ), ( - 'kubectl wait -n default --for=delete pod -l ' + 'kubectl --kubeconfig temp1 wait -n default --for=delete pod -l ' 'forge-namespace=forge-perry-1659078000', RunResult(0, b''), ), ( - 'kubectl apply -n default -f temp1', + 'kubectl --kubeconfig temp1 apply -n default -f temp2', RunResult(0, b''), ), ( - 'kubectl wait -n default --timeout=5m --for=condition=Ready ' + 'kubectl --kubeconfig temp1 wait -n default --timeout=5m --for=condition=Ready ' 'pod/forge-perry-1659078000-1659078000-banana', RunResult(0, b''), ), ( - 'kubectl logs -n default -f forge-perry-1659078000-1659078000-banana', + 'kubectl --kubeconfig temp1 logs -n default -f forge-perry-1659078000-1659078000-banana', RunResult(0, b''), ), ( - 'kubectl get pod -n default forge-perry-1659078000-1659078000-banana -o ' + 'kubectl --kubeconfig temp1 get pod -n default forge-perry-1659078000-1659078000-banana -o ' "jsonpath='{.status.phase}'", RunResult(0, b''), ), ( - 'kubectl get pods -n forge-perry-1659078000', + 'kubectl --kubeconfig temp1 get pods -n forge-perry-1659078000', RunResult(0, b''), ) ])) @@ -707,6 +735,7 @@ def testMain(self) -> None: lambda *_: FakeConfigBackend({ "enabled_clusters": ["forge-big-1"], "all_clusters": ["forge-big-1", "banana"], + "test_suites": {}, }) ) ) @@ -729,6 +758,7 @@ def testMain(self) -> None: "--github-server-url", "None", "--github-repository", "None", "--github-run-id", "None", + "banana-test", ], catch_exceptions=False, ) @@ -908,4 +938,50 @@ def testValidateMissingClusterConfig(self) -> None: "all_clusters": ["banana", "potato"], }), [], - ) \ No newline at end of file + ) + + def testClusterDelete(self) -> None: + runner = CliRunner() + shell = SpyShell(OrderedDict([ + ( + 'aws s3api get-object --bucket forge-wrapper-config --key ' + 'forge-wrapper-config.json temp1', + RunResult(0, b'') + ), + ( + 'aws s3api put-object --bucket forge-wrapper-config --key ' + 'forge-wrapper-config.json --body temp2', + RunResult(0, b'') + ), + ])) + clusters_before = { + "enabled_clusters": ["banana"], + "all_clusters": ["banana", "apple"], + } + clusters_after = { + **clusters_before, + "all_clusters": ["banana"], + } + filesystem = SpyFilesystem( + { + "temp2": json.dumps(clusters_after).encode(), + }, + { + "temp1": json.dumps(clusters_before).encode(), + }, + ) + with ExitStack() as stack: + stack.enter_context( + patch.object(forge, "LocalShell", lambda: shell) + ) + stack.enter_context( + patch.object(forge, "LocalFilesystem", lambda: filesystem) + ) + runner.invoke( + main, + ["config", "cluster", "delete", "apple"], + catch_exceptions=False, + ) + shell.assert_commands(self) + filesystem.assert_reads(self) + filesystem.assert_writes(self)