[forwardport] ci/cd: publish binaries onto a release when we create a…

… tag (ooni#609) (ooni#611) This diff forwardports cc3a2f1862dfbff4846a536468cdfe37bca92534 to the master branch Original commit message: - - - This pull request changes `mk` and github workflows to build and publish binaries on tag. We also update the documentation to explain this new branching model. Basically, we have release branches where we produce binary packages and we add extra code, on tag, to publish such packages inside a release. We discussed removing most secrets from builds in this repository and having a different tool/repository that takes in input also secrets for doing follow-up actions after publishing. As a consequence, this pull request also removes all pieces of code that require secrets. The next step is to reinstate this code in this new repository/tool. The existing code in `mk` also implemented caching. This feature was useful when doing local builds because it reduced the time required to obtain binary releases. With builds running as part of GitHub actions, we don't need caching because we spawn parallel machines to build binaries. Therefore, let us also remove caching, which makes the code simpler. (Caching in itself is hard and in ooni/probe#1875 I noted that, for example, caching of the `ooni/go` repository was leading to some unwanted behaviour when changing the branch. Without caching, this behaviour is gone and we always generally use fresh information to produce builds.) Of course, this means that local builds are now slower, but I do not think this is a problem _because_ we want to use GitHub actions for building in the common case. Reference issues: ooni/probe#1879 and ooni/probe#1875. The final aspect to mention to conclude this description is an implementation one: ``` gh release create -p $tag --target $GITHUB_SHA || true ``` The code above uses `|| true` because there could already be a release. So, basically, it means that, if a release does not already exist, then we're going to create one. Otherwise, it does not matter because there's already a release.
ainghazal · Nov 23, 2021 · 1ff4abc · 1ff4abc
1 parent 28ee018
commit 1ff4abc
Show file tree

Hide file tree

Showing 49 changed files with 894 additions and 1,951 deletions.
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
@@ -3,10 +3,12 @@ name: android
 on:
   push:
     branches:
-      - "mobile-staging"
       - "release/**"
+    tags:
+      - "v*"
+
 jobs:
-  test:
+  build_and_publish:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/setup-go@v1
@@ -23,4 +25,14 @@ jobs:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
 
-      - run: ./mk ./MOBILE/android/oonimkall.aar
+      - run: ./mk ./MOBILE/android
+
+      - run: |
+          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          gh release create -p $tag --target $GITHUB_SHA || true
+          gh release upload $tag --clobber ./MOBILE/android/oonimkall.aar \
+                                           ./MOBILE/android/oonimkall-sources.jar \
+                                           ./MOBILE/android/oonimkall.pom
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml
@@ -3,10 +3,12 @@ name: ios
 on:
   push:
     branches:
-      - "mobile-staging"
       - "release/**"
+    tags:
+      - "v*"
+
 jobs:
-  test:
+  build_and_publish:
     runs-on: macos-10.15
     steps:
       - uses: actions/setup-go@v1
@@ -23,4 +25,13 @@ jobs:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
 
-      - run: ./mk XCODE_VERSION=12.4 ./MOBILE/ios/oonimkall.xcframework.zip
+      - run: ./mk XCODE_VERSION=12.4 ./MOBILE/ios
+
+      - run: |
+          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          gh release create -p $tag --target $GITHUB_SHA || true
+          gh release upload $tag --clobber ./MOBILE/ios/oonimkall.xcframework.zip \
+                                           ./MOBILE/ios/oonimkall.podspec
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -4,10 +4,11 @@ on:
   push:
     branches:
       - "release/**"
-      - "ooniprobe-staging"
+    tags:
+      - "v*"
 
 jobs:
-  build_386:
+  build_and_publish_386:
     runs-on: "ubuntu-20.04"
     steps:
       - uses: actions/checkout@v2
@@ -19,16 +20,18 @@ jobs:
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
-      - run: ./mk DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/386
-      - run: ./E2E/ooniprobe.sh ./CLI/linux/386/ooniprobe
-      - run: ./CLI/linux/pubdebian
-        if: github.ref == 'refs/heads/ooniprobe-staging'
+      - run: ./mk ./CLI/ooniprobe-linux-386
+      - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386
+      - run: |
+          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          gh release create -p $tag --target $GITHUB_SHA || true
+          gh release upload $tag --clobber ./CLI/ooniprobe-linux-386
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
 
-  build_amd64:
+  build_and_publish_amd64:
     runs-on: "ubuntu-20.04"
     steps:
       - uses: actions/checkout@v2
@@ -40,16 +43,17 @@ jobs:
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
-      - run: ./mk DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/amd64
-      - run: ./E2E/ooniprobe.sh ./CLI/linux/amd64/ooniprobe
-      - run: ./CLI/linux/pubdebian
-        if: github.ref == 'refs/heads/ooniprobe-staging'
+      - run: ./mk ./CLI/ooniprobe-linux-amd64
+      - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-amd64
+      - run: |
+          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          gh release create -p $tag --target $GITHUB_SHA || true
+          gh release upload $tag --clobber ./CLI/ooniprobe-linux-amd64
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  build_arm:
+  build_and_publish_armv7:
     runs-on: "ubuntu-20.04"
     steps:
       - uses: actions/checkout@v2
@@ -63,16 +67,17 @@ jobs:
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
-      - run: ./mk DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm
-      - run: ./E2E/ooniprobe.sh ./CLI/linux/arm/ooniprobe
-      - run: ./CLI/linux/pubdebian
-        if: github.ref == 'refs/heads/ooniprobe-staging'
+      - run: ./mk ./CLI/ooniprobe-linux-armv7
+      - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv7
+      - run: |
+          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          gh release create -p $tag --target $GITHUB_SHA || true
+          gh release upload $tag --clobber ./CLI/ooniprobe-linux-armv7
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  build_arm64:
+  build_and_publish_arm64:
     runs-on: "ubuntu-20.04"
     steps:
       - uses: actions/checkout@v2
@@ -86,11 +91,12 @@ jobs:
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
-      - run: ./mk DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm64
-      - run: ./E2E/ooniprobe.sh ./CLI/linux/arm64/ooniprobe
-      - run: ./CLI/linux/pubdebian
-        if: github.ref == 'refs/heads/ooniprobe-staging'
+      - run: ./mk ./CLI/ooniprobe-linux-arm64
+      - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-arm64
+      - run: |
+          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          gh release create -p $tag --target $GITHUB_SHA || true
+          gh release upload $tag --clobber ./CLI/ooniprobe-linux-arm64
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -4,9 +4,12 @@ on:
   push:
     branches:
       - "release/**"
+    tags:
+      - "v*"
+
 jobs:
-  build:
-    runs-on: "macos-10.15"
+  build_and_publish:
+    runs-on: "macos-11"
     steps:
       - uses: actions/setup-go@v1
         with:
@@ -20,5 +23,14 @@ jobs:
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
-      - run: ./mk ./CLI/darwin/amd64/ooniprobe
-      - run: ./E2E/ooniprobe.sh ./CLI/darwin/amd64/ooniprobe
+      - run: ./mk ./CLI/ooniprobe-darwin
+      - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-darwin-amd64
+
+      - run: |
+          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          gh release create -p $tag --target $GITHUB_SHA || true
+          gh release upload $tag --clobber ./CLI/ooniprobe-darwin-amd64 \
+                                           ./CLI/ooniprobe-darwin-arm64
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/oohelperd.yml b/.github/workflows/oohelperd.yml
@@ -4,10 +4,11 @@ on:
   push:
     branches:
       - "release/**"
-      - "oohelperd-staging"
+    tags:
+      - "v*"
 
 jobs:
-  build_amd64:
+  build_and_publish:
     runs-on: "ubuntu-20.04"
     steps:
       - uses: actions/checkout@v2
@@ -16,35 +17,12 @@ jobs:
           go-version: "1.17.3"
 
       - name: build oohelperd binary
-        run: GOOS=linux GOARCH=amd64 go build -v ./internal/cmd/oohelperd
+        run: GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ./CLI/oohelperd-linux-amd64 -v -tags netgo -ldflags="-s -w -extldflags -static" ./internal/cmd/oohelperd
 
-      - run: find . -type f -name oohelperd
-
-      - name: install deps
-        run: |
-          sudo apt-get -q update
-          sudo apt-get install -yq --no-install-recommends curl devscripts \
-            dpkg-dev debhelper git python3 python3-requests python3-gnupg s3cmd
-
-      - name: update the debian changelog
-        run: |
-          version="$(go run ./internal/cmd/printversion)~$GITHUB_RUN_NUMBER"
-          cd ./internal/cmd/oohelperd
-          dch -v "$version" "New version ${version}"
-
-      - name: build deb package
-        run: |
-          cd ./internal/cmd/oohelperd
-          dpkg-buildpackage -us -uc -b
-
-      - name: upload package
-        if: github.ref == 'refs/heads/oohelperd-staging'
-        run: |
-          curl -fsSLO https://raw.githubusercontent.com/ooni/sysadmin/master/tools/debops-ci
-          chmod +x debops-ci
-          find . -name '*.deb'
-          ./debops-ci --show-commands upload --bucket-name ooni-internal-deb --arch "amd64" internal/cmd/*.deb
+      - run: |
+          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          gh release create -p $tag --target $GITHUB_SHA || true
+          gh release upload $tag --clobber ./CLI/oohelperd-linux-amd64
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -4,6 +4,9 @@ on:
   push:
     branches:
       - "release/**"
+    tags:
+      - "v*"
+
 jobs:
   build:
     runs-on: "ubuntu-20.04"
@@ -21,17 +24,42 @@ jobs:
         env:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
-      - run: ./mk MINGW_W64_VERSION="9.3-win32" ./CLI/windows/amd64/ooniprobe.exe
+      - run: ./mk MINGW_W64_VERSION="9.3-win32" ./CLI/ooniprobe-windows
       - uses: actions/upload-artifact@v2
         with:
-          name: ooniprobe.exe
-          path: ./CLI/windows/amd64/ooniprobe.exe
+          name: ooniprobe-windows-amd64.exe
+          path: ./CLI/ooniprobe-windows-amd64.exe
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ooniprobe-windows-386.exe
+          path: ./CLI/ooniprobe-windows-386.exe
+
   test:
     needs: build
     runs-on: "windows-2019"
     steps:
       - uses: actions/checkout@v2
       - uses: actions/download-artifact@v2
         with:
-          name: ooniprobe.exe
-      - run: bash.exe ./E2E/ooniprobe.sh ./ooniprobe.exe
+          name: ooniprobe-windows-amd64.exe
+      - run: bash.exe ./E2E/ooniprobe.sh ./ooniprobe-windows-amd64.exe
+
+  publish:
+    needs: test
+    runs-on: "ubuntu-20.04"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: ooniprobe-windows-amd64.exe
+      - uses: actions/download-artifact@v2
+        with:
+          name: ooniprobe-windows-386.exe
+      - run: |
+          tag=$(echo $GITHUB_REF | sed 's|refs/tags/||g')
+          gh release create -p $tag --target $GITHUB_SHA || true
+          gh release upload $tag --clobber ooniprobe-windows-386.exe \
+                                           ooniprobe-windows-amd64.exe
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/CLI/.gitignore b/CLI/.gitignore
@@ -1 +1,2 @@
 /miniooni-*
+/ooniprobe-*
diff --git a/CLI/linux/build → CLI/build-linux b/CLI/linux/build → CLI/build-linux
@@ -14,8 +14,11 @@ apk add --no-progress gcc git linux-headers musl-dev
 # some of the following exports are redundant but are however
 # useful because they provide explicit logging
 export GOARM=$GOARM
-export GOPATH=$GOPATH
 export CGO_ENABLED=1
 export GOOS=linux
 export GOARCH=$GOARCH
-go build -o "./CLI/linux/$GOARCH/" -ldflags='-s -w -extldflags "-static"' "$@" ./cmd/ooniprobe
+archname=$GOARCH
+if [ "$GOARCH" = "arm" -a "$GOARM" = "7" ]; then
+	archname="armv7"
+fi
+go build -o "./CLI/ooniprobe-linux-$archname" -ldflags='-s -w -extldflags "-static"' "$@" ./cmd/ooniprobe
diff --git a/CLI/darwin/amd64/.gitignore b/CLI/darwin/amd64/.gitignore
diff --git a/CLI/darwin/arm64/.gitignore b/CLI/darwin/arm64/.gitignore
diff --git a/CLI/linux/386/.gitignore b/CLI/linux/386/.gitignore
diff --git a/CLI/linux/amd64/.gitignore b/CLI/linux/amd64/.gitignore
diff --git a/CLI/linux/arm/.gitignore b/CLI/linux/arm/.gitignore
diff --git a/CLI/linux/arm64/.gitignore b/CLI/linux/arm64/.gitignore