-
Notifications
You must be signed in to change notification settings - Fork 33
/
setup
executable file
·119 lines (105 loc) · 6.03 KB
/
setup
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
#!/usr/bin/env bash
set -x # Show commands
set -eu # Errors/undefined vars are fatal
set -o pipefail # Check all commands in a pipeline
date
# NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE
#
# >>> mozilla-central is special because we like code coverage data! <<<
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
#
# Unless a `TRYPUSH_REV` is specified below, we are going to use the
# `project.relman.code-coverage.production.repo.mozilla-central.latest` route
# in order to try and find a recent hg revision to use that has coverage.
#
# To do this, we are going to use a few taskcluster APIs directly via curl.
# Arguably, a better idea is to use the `taskcluster` binary, but we only
# actually need to hit 2 URLs here and so the moving parts theory favors not
# provisioning `taskcluster` yet, but it would probably be a reasonable step.
# This defaults to the most recent searchfox indexing run for mozilla-central.
REVISION_TREE=mozilla-central
REVISION_ID=latest
TC_ROOT_URL=https://firefox-ci-tc.services.mozilla.com
TC_INDEX_API_URL=${TC_ROOT_URL}/api/index/v1/task
COVERAGE_ROUTE=project.relman.code-coverage.production.repo.${REVISION_TREE}.latest
TC_QUEUE_API_URL=${TC_ROOT_URL}/api/queue/v1/task
# Set TRYPUSH_REV in the environment to e.g. ee64db93dcc149da9313460317257b8c42eec5b2
# or whatever to test a try revision. It needs to be the full rev hash, so that we can
# find the artifacts in taskcluster.
TRYPUSH_REV=${TRYPUSH_REV:-}
if [ -n "$TRYPUSH_REV" ]; then
REVISION_TREE=try
REVISION_ID="revision.${TRYPUSH_REV}"
echo "TRYPUSH_REV was used, ignoring coverage-based revision choice. Using: ${REVISION_ID}"
else
# find the taskId that the "latest" code coverage for our tree
COV_TASKID=$(curl -ssL "${TC_INDEX_API_URL}/${COVERAGE_ROUTE}" | jq -Mr '.taskId')
# get the task definition JSON and save it into a variable so we can pick
# multiple fields out
COV_TASK_INFO=$(curl -ssL "${TC_QUEUE_API_URL}/${COV_TASKID}")
# The revision to index is passed specifically to the job
COV_HG_REV=$(jq -Mr '.payload.env.REVISION' <<< $COV_TASK_INFO)
# How many whole days have passed since the coverage aggregation task was
# started and now? If it's been less than a day (therefore "0"), the data
# is good enough for our purposes.
if [[ $COV_HG_REV = "null" ]]; then
# In the failure mode we're seeing in Bug 1909053, COV_TASK_INFO is a
# 404 JSON blob and we will have extracted null from it. Let's just
# pick a very obvious non-zero value so that we can just use the latest
# indexed revision.
echo "No sign of coverage data; pretending the data is stale."
COV_RECENCY_DAYS=999
else
COV_RECENCY_DAYS=$(jq -Mr '.created | sub("\\.[0-9]+Z$"; "Z") | (now - fromdate) / (24 * 60 * 60) | floor' <<< $COV_TASK_INFO)
fi
if [[ $COV_RECENCY_DAYS = "0" ]]; then
# We also need to check that the revision is itself recent enough, see
# bug 1818083.
COV_HG_REV_TARGET_INFO=$(curl -SsfL --compressed "${TC_INDEX_API_URL}/gecko.v2.${REVISION_TREE}.revision.${COV_HG_REV}.firefox.linux64-searchfox-debug/artifacts/public/build/target.json")
COV_BUILDID=$(jq -Mr .buildid <<< $COV_HG_REV_TARGET_INFO)
COV_BUILDID_IS_RECENT=$(jq -Mr --arg today $(date +%Y%m%d) --arg yesterday $(date --date yesterday +%Y%m%d) '.buildid | startswith($today) or startswith($yesterday)' <<< $COV_HG_REV_TARGET_INFO)
if [ $COV_BUILDID_IS_RECENT = "true" ]; then
REVISION_ID="revision.${COV_HG_REV}"
echo "Coverage data is recent enough, using explicit revision: ${REVISION_ID}"
else
echo "Coverage data is recent, but it's for an old revision (${COV_HG_REV}, ${COV_BUILDID}), sticking with: ${REVISION_ID}"
fi
else
echo "Coverage data is ${COV_RECENCY_DAYS} old, sticking with: ${REVISION_ID}"
fi
fi
# The next line populates the INDEXED_HG_REV and PREEXISTING_HG_REV env vars.
# Note that PREEXISTING_HG_REV will generally be empty in production.
source $CONFIG_REPO/shared/resolve-gecko-revs.sh $REVISION_TREE $REVISION_ID
$CONFIG_REPO/shared/checkout-gecko-repos.sh $REVISION_TREE "master" "$INDEXED_HG_REV"
$CONFIG_REPO/shared/fetch-tc-artifacts.sh $REVISION_TREE $INDEXED_HG_REV "$PREEXISTING_HG_REV"
date
# Also generate blame for some other branches, because the gecko-blame repo is
# shared by those branches. We do this here instead of in e.g. ../mozilla-beta/setup
# because it's best to have only one indexer instance responsible for updating and
# pushing the tarball to S3, to avoid accidental clobbers. It's not a great situation
# architecturally, but it's better for performance.
#
# Base case for dynamic branch creation, see below.
LASTBRANCH="master"
# If you are adding branches, add them after the branch they forked off of,
# putting the branch at the front of the list if they forked off of m-c. So
# in the case of new ESR branches (which fork off of "release"), they go after
# release and before existing ESR branches.
# Note: the pine branch used to be in here, but was removed in bug 1841395.
# If the pine branch gets reset for a new project and we want to
# add it back in here, we might need to manually update the tarball to
# strip the existing pine branch. I don't know if it will work otherwise.
for BRANCH in elm cedar cypress beta release esr128 esr115 esr102 esr91 esr78 esr68 esr60 esr45 esr31 esr17; do
echo "Updating gecko-dev branch $BRANCH to latest from cinnabar upstream..."
pushd $GIT_ROOT
git branch -f $BRANCH $BRANCH/branches/default/tip
popd
echo "Generating blame information for $BRANCH..."
# Check if there's a branch in the blame repo yet, and if there isn't, fork
# it from the previous branch in the list, with a base case of "master".
git -C $BLAME_ROOT show-ref --verify --quiet "refs/heads/${BRANCH}" || git -C $BLAME_ROOT branch "${BRANCH}" "${LASTBRANCH}"
BLAME_REF="refs/heads/$BRANCH" $MOZSEARCH_PATH/tools/target/release/build-blame $GIT_ROOT $BLAME_ROOT
LASTBRANCH="${BRANCH}"
done
date