triage: prepare for alternate projects, datasets, buckets

config/jobs/kubernetes/wg-k8s-infra/trusted/wg-k8s-infra-test-infra.yaml

@@ -95,6 +95,14 @@ periodics:
       # TODO: determine the optimal number of workers, 2*CPU-1 is an assumption
       - name: NUM_WORKERS
         value: "13"
+      - name: TRIAGE_DATASET_TABLE
+        value: "k8s-gubernator:temp.triage" # TODO: definitely don't have permission to write here, need our own bq dataset
+      - name: TRIAGE_TEMP_GCS_PATH
+        value: "gs://k8s-triage/triage_tests"
+      - name: TRIAGE_GCS_PATH
+        value: "gs://k8s-triage"
+      - name: TRIAGE_BQ_USAGE_PROJECT
+        value: "k8s-infra-prow-build-trusted"
       command:
       - timeout
       args:

triage/README.md

@@ -200,4 +200,4 @@ To update the triage image run `make push` from `./triage` which will trigger a
 To update Triage frontend in Production or Staging manually run `make push-static` or `make push-staging` respectively. Otherwise it is updated on postsubmit via [post-test-infra-upload-triage](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes/test-infra/test-infra-trusted.yaml#L616).
 
 ### Staging
-   To acces staging see [Triage Staging](https://storage.googleapis.com/k8s-gubernator/triage/staging).
+   To acces staging see [Triage Staging](https://storage.googleapis.com/k8s-triage/staging).

triage/interactive.js

@@ -303,7 +303,7 @@ function getData() {
     setElementVisibility('btn-sig-group', false);
   }
 
-  var url = '/k8s-gubernator/triage/';
+  var url = '/triage/';
   if (document.location.host == 'storage.googleapis.com' && document.location.pathname.endsWith('index.html')) {
     // Use the bucket name where available
     var pathname = document.location.pathname;

triage/update_summaries.sh

@@ -15,20 +15,39 @@
 # limitations under the License.
 
 set -exu
-cd $(dirname $0)
+
+# dataset table to query for build info
+readonly BUILD_DATASET_TABLE="${BUILD_DATASET_TABLE:-"k8s-gubernator:build.all"}"
+
+# dataset to write temp results to for triage
+readonly TRIAGE_DATASET_TABLE="${TRIAGE_DATASET_TABLE:-"k8s-gubernator:temp.triage"}"
+
+# gcs bucket to write temporary results to
+readonly TRIAGE_TEMP_GCS_PATH="${TRIAGE_TEMP_GCS_PATH:-"gs://k8s-gubernator/triage_tests"}"
+
+# gcs uri to write final triage results to
+readonly TRIAGE_GCS_PATH="${TRIAGE_GCS_PATH:-"gs://k8s-gubernator/triage"}"
+
+# the gcp project against which to bill bq usage
+readonly TRIAGE_BQ_USAGE_PROJECT="${TRIAGE_BQ_USAGE_PROJECT:-"k8s-gubernator"}"
+
+cd "$(dirname "$0")"
 
 start=$(date +%s)
 
 if [[ -e ${GOOGLE_APPLICATION_CREDENTIALS-} ]]; then
+  echo "activating service account with credentials at: ${GOOGLE_APPLICATION_CREDENTIALS}"
   gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
 fi
 
-gcloud config set project k8s-gubernator
+gcloud config set project "${TRIAGE_BQ_USAGE_PROJECT}"
+
 bq show <<< $'\n'
 
 date
 
-bq --headless --format=json query --max_rows 1000000 \
+# populate triage_builds.json with build metadata
+bq --project_id="${TRIAGE_BQ_USAGE_PROJECT}" --headless --format=json query --max_rows 1000000 \
   "select
     path,
     timestamp_to_sec(started) started,
@@ -40,32 +59,33 @@ bq --headless --format=json query --max_rows 1000000 \
     job,
     number
   from
-    [k8s-gubernator:build.all]
+    [${BUILD_DATASET_TABLE}]
   where
     timestamp_to_sec(started) > TIMESTAMP_TO_SEC(DATE_ADD(CURRENT_DATE(), -14, 'DAY'))
     and job != 'ci-kubernetes-coverage-unit'" \
   > triage_builds.json
 
-bq query --allow_large_results --headless --max_rows 0 --replace --destination_table k8s-gubernator:temp.triage \
+# populate ${TRIAGE_DATASET_TABLE} with test failures
+bq --project_id="${TRIAGE_BQ_USAGE_PROJECT}" query --allow_large_results --headless --max_rows 0 --replace --destination_table "${TRIAGE_DATASET_TABLE}" \
   "select
     timestamp_to_sec(started) started,
     path build,
     test.name name,
     test.failure_text failure_text
   from
-    [k8s-gubernator:build.all]
+    [${BUILD_DATASET_TABLE}]
   where
     test.failed
     and timestamp_to_sec(started) > TIMESTAMP_TO_SEC(DATE_ADD(CURRENT_DATE(), -14, 'DAY'))
     and job != 'ci-kubernetes-coverage-unit'"
 
-gsutil rm gs://k8s-gubernator/triage_tests/shard_*.json.gz || true
-bq extract --compression GZIP --destination_format NEWLINE_DELIMITED_JSON 'k8s-gubernator:temp.triage' gs://k8s-gubernator/triage_tests/shard_*.json.gz
+gsutil rm "${TRIAGE_TEMP_GCS_PATH}/shard_*.json.gz" || true
+bq extract --compression GZIP --destination_format NEWLINE_DELIMITED_JSON "${TRIAGE_DATASET_TABLE}" "${TRIAGE_TEMP_GCS_PATH}/shard_*.json.gz"
 mkdir -p triage_tests
-gsutil cp -r gs://k8s-gubernator/triage_tests/* triage_tests/
+gsutil cp -r "${TRIAGE_TEMP_GCS_PATH}/*" triage_tests/
 gzip -df triage_tests/*.gz
 
-# gsutil cp gs://k8s-gubernator/triage/failure_data.json failure_data_previous.json
+# gsutil cp "${TRIAGE_BUCKET}/failure_data.json failure_data_previous.json
 
 mkdir -p slices
 
@@ -80,10 +100,10 @@ gsutil_cp() {
   gsutil -h 'Cache-Control: no-store, must-revalidate' -m cp -Z -a public-read "$@"
 }
 
-gsutil_cp failure_data.json gs://k8s-gubernator/triage/
-gsutil_cp slices/*.json gs://k8s-gubernator/triage/slices/
-gsutil_cp failure_data.json "gs://k8s-gubernator/triage/history/$(date -u +%Y%m%d).json"
+gsutil_cp failure_data.json "${TRIAGE_GCS_PATH}/"
+gsutil_cp slices/*.json "${TRIAGE_GCS_PATH}/slices/"
+gsutil_cp failure_data.json "${TRIAGE_GCS_PATH}/history/$(date -u +%Y%m%d).json"
 
 stop=$(date +%s)
-elapsed=$(( ${stop} - ${start} ))
-echo "Finished in $(( ${elapsed} / 60))m$(( ${elapsed} % 60))s"
+elapsed=$(( stop - start ))
+echo "Finished in $(( elapsed / 60))m$(( elapsed % 60))s"