Merge pull request #107 from datawire/rlane/remove-M1-support

Remove CodeMagic and M1 support

.github/workflows/github-runner-provisioner.yaml

@@ -17,7 +17,6 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       GITHUB_TOKEN: FAKE_TOKEN
-      CODEMAGIC_TOKEN: FAKE_TOKEN
       WEBHOOK_TOKEN: FAKE_TOKEN
     steps:
       - uses: actions/checkout@v4
@@ -33,7 +32,6 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       GITHUB_TOKEN: ${{ secrets.GRP_GITHUB_TOKEN }}
-      CODEMAGIC_TOKEN: FAKE_TOKEN
       WEBHOOK_TOKEN: FAKE_TOKEN
       AWS_ACCESS_KEY_ID: ${{ secrets.GRP_AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.GRP_AWS_SECRET_ACCESS_KEY }}
@@ -54,37 +52,14 @@ jobs:
             type=gha,scope=refs/heads/main
             type=gha,scope=github-runner-provisioner-${{github.ref}}
           cache-to: type=gha,mode=max,scope=github-runner-provisioner-${{github.ref}}
-      - name: Test CodeMagic
-        run: |
-          cd github-runner-provisioner
-          docker load --input /tmp/github-runner-provisioner.tar
-          docker run -d --name=github-runner-provisioner-codemagic -p 8080:8080 \
-            -e GITHUB_TOKEN=${GITHUB_TOKEN} \
-            -e CODEMAGIC_TOKEN=${CODEMAGIC_TOKEN} \
-            -e WEBHOOK_TOKEN=${WEBHOOK_TOKEN} \
-            -e USE_CODEMAGIC="true" \
-            datawiredev/github-runner-provisioner:${{ github.sha }}
-          make test-macOS-arm64
-          docker stop github-runner-provisioner-codemagic
-          echo
-          echo "Container logs:"
-          docker logs github-runner-provisioner-codemagic
-          echo
-          if [ "$(cat /tmp/test-macOS-arm64)" != "200" ]; then
-            echo "Test failed"
-            exit 1
-          fi
-          echo "Test Successful"
 
       - name: Test AWS
         run: |
           cd github-runner-provisioner
           docker load --input /tmp/github-runner-provisioner.tar
           docker run -d --name=github-runner-provisioner-aws -p 8080:8080 \
             -e GITHUB_TOKEN=${GITHUB_TOKEN} \
-            -e CODEMAGIC_TOKEN=${CODEMAGIC_TOKEN} \
             -e WEBHOOK_TOKEN=${WEBHOOK_TOKEN} \
-            -e USE_CODEMAGIC="false" \
             -e AWS_REGION="us-east-1" \
             -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
             -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
@@ -95,7 +70,7 @@ jobs:
           echo "Container logs:"
           docker logs github-runner-provisioner-aws
           echo
-          if [ $(cat /tmp/test-ubuntu-arm64) != "200" ] || [ $(cat /tmp/test-macOS-arm64) != "200" ]; then
+          if [ $(cat /tmp/test-ubuntu-arm64) != "200" ]; then
             echo "Test failed"
             exit 1
           fi
@@ -110,8 +85,8 @@ jobs:
       - uses: docker/setup-buildx-action@v3
       - uses: docker/login-action@v3
         with:
-            username: ${{ secrets.DOCKERHUB_USERNAME }}
-            password: ${{ secrets.DOCKERHUB_TOKEN }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - run: cd github-runner-provisioner
       - name: Build and push
         uses: docker/build-push-action@v5

docs/ACTION_RUNNERS.md

@@ -1,6 +1,6 @@
 # Custom GitHub action runners
 
-There are self-hosted Mac M1 and Ubuntu ARM64 runners available for GitHub actions. There runners are EC2 instances hosted in AWS.
+Self-hosted Ubuntu ARM64 runners are available for GitHub actions. These runners are EC2 instances hosted in AWS.
 
 In the future, we may make additional runners available depending on the needs of the different teams.
 
@@ -18,26 +18,6 @@ Before a job can use a self-hosted runner, the following settings need to be con
 
 Once the webhook is configured, you can use the runners as described below.
 
-## Mac M1 runners
-
-There are self-hosted Mac M1 (ARM64) runners that can be used in a workflow by using `runs-on: macOS-arm64`.
-
-```yaml
-jobs:
-  my_job:
-    runs-on: macOS-arm64
-    steps:
-      # The provision-cluster action will automatically register a cleanup hook to remove the
-      # cluster it provisions when the job is done.
-      - uses: actions/checkout@v4
-```
-
-The following limitations apply to Mac M1 runners:
-
-- It will take between 30 minutes and up to 3 hours for a runner to be available from the moment it is requested by a job.
-- There is a limit of 10 active Mac M1 runners. Any build that requests a Mac M1 during this time will stay in a queued state until a runner is available. If a job is queued for more than 24 hours, it will be marked as failed.
-- Once a Mac M1 runner is created, it will continue to run for up to 24 hours, picking-up oe or more jobs. What the means is that jobs are responsible for ensuring that runners are in a clean state before they are used.
-
 ## Ubuntu ARM64 runners
 
 These self-hosted runners are created on-demand. It takes about a minute for the runner to be available, and once the job finishes, they are destroyed.
@@ -49,7 +29,5 @@ jobs:
   my_job:
     runs-on: ubuntu-arm64
     steps:
-      # The provision-cluster action will automatically register a cleanup hook to remove the
-      # cluster it provisions when the job is done.
       - uses: actions/checkout@v4
 ```

github-runner-provisioner/Makefile

@@ -3,8 +3,8 @@ DRY_RUN := true
 RUNNER_TAG := ''
 
 .PHONY: test-runners
-test-runners: test-ubuntu-arm64 test-macOS-arm64
-	@if [ "$$(cat /tmp/test-ubuntu-arm64)" != "200" ] || [ "$$(cat /tmp/test-macOS-arm64)" != "200" ]; then \
+test-runners: test-ubuntu-arm64
+	@if [ "$$(cat /tmp/test-ubuntu-arm64)" != "200" ]; then \
 		@echo "Test failed"; \
 		exit 1; \
 	fi
@@ -18,29 +18,12 @@ test-ubuntu-arm64: /tmp/test-ubuntu-arm64
 /tmp/test-ubuntu-arm64:
 	make -s run-ubuntu-arm64 HOSTNAME=http://localhost:8080 DRY_RUN=true | tail -n 1 > /tmp/test-ubuntu-arm64
 
-.PHONY: test-macOS-arm64
-test-macOS-arm64: /tmp/test-macOS-arm64
-	@echo "Response code: $$(cat /tmp/test-macOS-arm64)"
-
-.PHONY: /tmp/test-macOS-arm64
-/tmp/test-macOS-arm64:
-	make -s run-macOS-arm64 HOSTNAME=http://localhost:8080 DRY_RUN=true | tail -n 1 > /tmp/test-macOS-arm64
-
 .PHONY: run-ubuntu-arm64
 run-ubuntu-arm64:
 	# SHA can be calculated like this:
     #echo -n "payload=$(cat github-runner-provisioner/test/ubuntu-arm64_payload.json)" | openssl dgst -sha1 -hmac FAKE_TOKEN
 	make test-github-provisioner SHA1=8e39e0658c5eacf3a3e006a46ef46092cbccb5ec RUNNER_TAG=ubuntu-arm64
 
-.PHONY: run-macOS-arm64
-run-macOS-arm64:
-	# SHA can be calculated like this:
-	#echo -n "payload=$(cat github-runner-provisioner/test/macOS-arm64_payload.json)" | openssl dgst -sha1 -hmac FAKE_TOKEN
-
-	make test-github-provisioner SHA1=e504cfa93721fbea2a394d4de9c9be7d5270fc19 RUNNER_TAG=macOS-arm64
-
-
-
 .PHONY: test-github-provisioner
 test-github-provisioner:
 	curl -v -o /dev/null -s $(HOSTNAME)/github-runner-provisioner/?dry-run=$(DRY_RUN) -d "payload=$$(cat test/$(RUNNER_TAG)_payload.json)" -H 'X-Hub-Signature-256: sha1=$(SHA1)' -w "%{http_code}\n"

github-runner-provisioner/README.md

@@ -6,16 +6,12 @@ This service is based on the [echo template](https://github.com/datawire/infrast
 
 We use the GitHub-Runner-Provisioner to serve a webhook to GitHub Actions. GitHub will send any Actions events to the GRP running in Skunkworks, which will parse those events looking for workflows that request special labels in their `runs-on` property.
 
-Using the GitHub Self-Hosted Runner binaries we then spin up the custom runners in one of our supported runner providers - currently AWS and CodeMagic. Supported runners are configured in [runner.go](runner.go).
+Using the GitHub Self-Hosted Runner binaries we then spin up the custom runners in one of our supported runner providers - currently AWS only. Supported runners are configured in [runner.go](runner.go).
 
 ### AWS
 
 AWS runners are created in EC2 using the AWS SDK. See the [aws_runners](internal/aws/runners) package for details on the implementation.
 
-### CodeMagic
-
-CodeMagic runners are actually CodeMagic Builds (CI jobs in their service) that then pull the GitHub Self-Hosted binaries and register themselves as ephemeral (single-use) runners - picking up a single job from the calling repo and then terminating.
-
 ## Testing
 
 ### Integration Tests
@@ -24,12 +20,12 @@ CodeMagic runners are actually CodeMagic Builds (CI jobs in their service) that
 
 You will also need to set `GITHUB_TOKEN` to a PAT for the D6E Automaton. These values can all be found in the [github-runner-provisioner-secrets.yaml](/keybase/team/datawireio/skunkworks/github-runner-provisioner-secrets.yaml) file in Keybase - you will need to base64 decode them before use. If only running dry-runs only AWS and GitHub authentication is required.
 
-To test the application we use targets in the Makefile. The `make go-unit-tests` target will run the unit tests, and `make test-runners` will run the integration tests against the dry-run endpoints. Note that to test the AWS `macOS-arm64` runner you will need to set the `USE_CODEMAGIC` environment variable to `true` in the GRP.
+To test the application we use targets in the Makefile. The `make go-unit-tests` target will run the unit tests, and `make test-runners` will run the integration tests against the dry-run endpoints.
 
-Testing CodeMagic M1 & AWS ubuntu-arm64:
+Testing AWS ubuntu-arm64:
 
 ```bash
-USE_CODEMAGIC=true GITHUB_TOKEN=<pat> go run main.go --dry-run
+GITHUB_TOKEN=<pat> go run main.go --dry-run
 make test-runners
 ```
 
@@ -51,6 +47,4 @@ The runner provisioner requires the following variables to be configured:
   We use the `D6E-Automaton`'s token in production.
 - `WEBHOOK_TOKEN` - the secret used to configure the webhook in GitHub. We use the token stored at
   `/Keybase/team/datawireio/infra/github-runner-provisioner-secrets`
-- `CODEMAGIC_TOKEN` - the secret used to authenticate to the CodeMagic build API to trigger M1 runners
-- `USE_CODEMAGIC` - a boolean flag to indicate whether to use CodeMagic or AWS to provision M1 runners
 - AWS auth can be configured with `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` or by using the AWS CLI

github-runner-provisioner/config.go

@@ -6,10 +6,8 @@ import (
 )
 
 type Config struct {
-	GithubToken    string `required:"true" envconfig:"GITHUB_TOKEN"`
-	WebhookToken   string `required:"true" envconfig:"WEBHOOK_TOKEN"`
-	CodeMagicToken string `required:"true" envconfig:"CODEMAGIC_TOKEN"`
-	UseCodeMagic   bool   `default:"true"  envconfig:"USE_CODEMAGIC"`
+	GithubToken  string `required:"true" envconfig:"GITHUB_TOKEN"`
+	WebhookToken string `required:"true" envconfig:"WEBHOOK_TOKEN"`
 }
 
 func NewConfig() *Config {

github-runner-provisioner/github-runner-provisioner.yaml

@@ -21,7 +21,7 @@ kind: Service
 metadata:
   annotations:
     a8r.io/chat: '#team-infra'
-    a8r.io/description: "Mac M1 provisioner for GitHub jobs in Ambassador Labs repositories"
+    a8r.io/description: "GitHub runner provisioner for GitHub jobs in Ambassador repositories"
     a8r.io/logs: ""
     a8r.io/owner: Infrastructure
     a8r.io/repository: https://github.com/datawire/infra-actions
@@ -79,8 +79,6 @@ spec:
         env:
         - name: AWS_REGION
           value: us-east-1
-        - name: USE_CODEMAGIC
-          value: "true"
         envFrom:
           - secretRef:
               name: github-runner-provisioner-secrets
@@ -111,6 +109,7 @@ spec:
         resources:
           limits:
             cpu: 1000m
+            memory: 600Mi
           requests:
             cpu: 50m
             memory: 300Mi

github-runner-provisioner/internal/aws/runner.go

@@ -12,7 +12,6 @@ import (
 const dryRunApiError = "DryRunOperation"
 
 var runnerParams = map[string]func(string, string, string, string, bool) (ec2.RunInstancesInput, error){
-	"macOS-arm64":  aws_runners.MacM1RunInstancesInput,
 	"ubuntu-arm64": aws_runners.UbuntuArm64RunInstancesInput,
 }