diff --git a/config/jobs/kubernetes/wg-k8s-infra/trusted/wg-k8s-infra-test-infra.yaml b/config/jobs/kubernetes/wg-k8s-infra/trusted/wg-k8s-infra-test-infra.yaml index 8d27c8b21192..2df06c2dcda0 100644 --- a/config/jobs/kubernetes/wg-k8s-infra/trusted/wg-k8s-infra-test-infra.yaml +++ b/config/jobs/kubernetes/wg-k8s-infra/trusted/wg-k8s-infra-test-infra.yaml @@ -95,6 +95,14 @@ periodics: # TODO: determine the optimal number of workers, 2*CPU-1 is an assumption - name: NUM_WORKERS value: "13" + - name: TRIAGE_DATASET_TABLE + value: "k8s-gubernator:temp.triage" # TODO: definitely don't have permission to write here, need our own bq dataset + - name: TRIAGE_TEMP_GCS_PATH + value: "gs://k8s-triage/triage_tests" + - name: TRIAGE_GCS_PATH + value: "gs://k8s-triage" + - name: TRIAGE_BQ_USAGE_PROJECT + value: "k8s-infra-prow-build-trusted" command: - timeout args: diff --git a/triage/README.md b/triage/README.md index 82c9aff281dc..29302a1d8855 100644 --- a/triage/README.md +++ b/triage/README.md @@ -200,4 +200,4 @@ To update the triage image run `make push` from `./triage` which will trigger a To update Triage frontend in Production or Staging manually run `make push-static` or `make push-staging` respectively. Otherwise it is updated on postsubmit via [post-test-infra-upload-triage](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes/test-infra/test-infra-trusted.yaml#L616). ### Staging - To acces staging see [Triage Staging](https://storage.googleapis.com/k8s-gubernator/triage/staging). + To acces staging see [Triage Staging](https://storage.googleapis.com/k8s-triage/staging). diff --git a/triage/interactive.js b/triage/interactive.js index 025da7ff3025..cb08c85dcc9a 100644 --- a/triage/interactive.js +++ b/triage/interactive.js @@ -303,7 +303,7 @@ function getData() { setElementVisibility('btn-sig-group', false); } - var url = '/k8s-gubernator/triage/'; + var url = '/triage/'; if (document.location.host == 'storage.googleapis.com' && document.location.pathname.endsWith('index.html')) { // Use the bucket name where available var pathname = document.location.pathname; diff --git a/triage/update_summaries.sh b/triage/update_summaries.sh index d4be9fc53947..b144bcc8d626 100755 --- a/triage/update_summaries.sh +++ b/triage/update_summaries.sh @@ -15,20 +15,39 @@ # limitations under the License. set -exu -cd $(dirname $0) + +# dataset table to query for build info +readonly BUILD_DATASET_TABLE="${BUILD_DATASET_TABLE:-"k8s-gubernator:build.all"}" + +# dataset to write temp results to for triage +readonly TRIAGE_DATASET_TABLE="${TRIAGE_DATASET_TABLE:-"k8s-gubernator:temp.triage"}" + +# gcs bucket to write temporary results to +readonly TRIAGE_TEMP_GCS_PATH="${TRIAGE_TEMP_GCS_PATH:-"gs://k8s-gubernator/triage_tests"}" + +# gcs uri to write final triage results to +readonly TRIAGE_GCS_PATH="${TRIAGE_GCS_PATH:-"gs://k8s-gubernator/triage"}" + +# the gcp project against which to bill bq usage +readonly TRIAGE_BQ_USAGE_PROJECT="${TRIAGE_BQ_USAGE_PROJECT:-"k8s-gubernator"}" + +cd "$(dirname "$0")" start=$(date +%s) if [[ -e ${GOOGLE_APPLICATION_CREDENTIALS-} ]]; then + echo "activating service account with credentials at: ${GOOGLE_APPLICATION_CREDENTIALS}" gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" fi -gcloud config set project k8s-gubernator +gcloud config set project "${TRIAGE_BQ_USAGE_PROJECT}" + bq show <<< $'\n' date -bq --headless --format=json query --max_rows 1000000 \ +# populate triage_builds.json with build metadata +bq --project_id="${TRIAGE_BQ_USAGE_PROJECT}" --headless --format=json query --max_rows 1000000 \ "select path, timestamp_to_sec(started) started, @@ -40,32 +59,33 @@ bq --headless --format=json query --max_rows 1000000 \ job, number from - [k8s-gubernator:build.all] + [${BUILD_DATASET_TABLE}] where timestamp_to_sec(started) > TIMESTAMP_TO_SEC(DATE_ADD(CURRENT_DATE(), -14, 'DAY')) and job != 'ci-kubernetes-coverage-unit'" \ > triage_builds.json -bq query --allow_large_results --headless --max_rows 0 --replace --destination_table k8s-gubernator:temp.triage \ +# populate ${TRIAGE_DATASET_TABLE} with test failures +bq --project_id="${TRIAGE_BQ_USAGE_PROJECT}" query --allow_large_results --headless --max_rows 0 --replace --destination_table "${TRIAGE_DATASET_TABLE}" \ "select timestamp_to_sec(started) started, path build, test.name name, test.failure_text failure_text from - [k8s-gubernator:build.all] + [${BUILD_DATASET_TABLE}] where test.failed and timestamp_to_sec(started) > TIMESTAMP_TO_SEC(DATE_ADD(CURRENT_DATE(), -14, 'DAY')) and job != 'ci-kubernetes-coverage-unit'" -gsutil rm gs://k8s-gubernator/triage_tests/shard_*.json.gz || true -bq extract --compression GZIP --destination_format NEWLINE_DELIMITED_JSON 'k8s-gubernator:temp.triage' gs://k8s-gubernator/triage_tests/shard_*.json.gz +gsutil rm "${TRIAGE_TEMP_GCS_PATH}/shard_*.json.gz" || true +bq extract --compression GZIP --destination_format NEWLINE_DELIMITED_JSON "${TRIAGE_DATASET_TABLE}" "${TRIAGE_TEMP_GCS_PATH}/shard_*.json.gz" mkdir -p triage_tests -gsutil cp -r gs://k8s-gubernator/triage_tests/* triage_tests/ +gsutil cp -r "${TRIAGE_TEMP_GCS_PATH}/*" triage_tests/ gzip -df triage_tests/*.gz -# gsutil cp gs://k8s-gubernator/triage/failure_data.json failure_data_previous.json +# gsutil cp "${TRIAGE_BUCKET}/failure_data.json failure_data_previous.json mkdir -p slices @@ -80,10 +100,10 @@ gsutil_cp() { gsutil -h 'Cache-Control: no-store, must-revalidate' -m cp -Z -a public-read "$@" } -gsutil_cp failure_data.json gs://k8s-gubernator/triage/ -gsutil_cp slices/*.json gs://k8s-gubernator/triage/slices/ -gsutil_cp failure_data.json "gs://k8s-gubernator/triage/history/$(date -u +%Y%m%d).json" +gsutil_cp failure_data.json "${TRIAGE_GCS_PATH}/" +gsutil_cp slices/*.json "${TRIAGE_GCS_PATH}/slices/" +gsutil_cp failure_data.json "${TRIAGE_GCS_PATH}/history/$(date -u +%Y%m%d).json" stop=$(date +%s) -elapsed=$(( ${stop} - ${start} )) -echo "Finished in $(( ${elapsed} / 60))m$(( ${elapsed} % 60))s" +elapsed=$(( stop - start )) +echo "Finished in $(( elapsed / 60))m$(( elapsed % 60))s"