all: cache go module downloads and other build and test artifacts (#3727

)

.github/actions/setup-go/action.yml

@@ -0,0 +1,51 @@
+name: 'Setup the Go environment'
+description: 'Installs go and restores/saves the build/module cache'
+inputs:
+  go-version:
+    required: true
+runs:
+  using: "composite"
+  steps:
+  - name: Set up Go
+    uses: actions/setup-go@v2
+    with:
+      go-version: ${{ inputs.go-version }}
+      stable: ${{ !(contains(inputs.go-version, 'beta') || contains(inputs.go-version, 'rc')) }}
+
+  # Restore original modification time of files based on the date of the most
+  # recent commit that modified them as mtimes affect the Go test cache.
+  - name: Restore modification time of checkout files
+    shell: bash
+    run: |
+      # Set a base, fixed modification time of all directories.
+      # git-restore-mtime doesn't set the mtime of of all directories.
+      # (see https://github.com/MestreLion/git-tools/issues/47 for details)
+      touch -m -t '201509301646' $(find . -type d -not -path '.git/*')
+      # Restore original modification time from git. git clone sets the
+      # modification time to the current time, but Go tests that access fixtures
+      # get invalidated if their modification times change.
+      sudo apt-get install -y git-restore-mtime
+      git restore-mtime
+
+  # The PREFIX must uniquely identify the specific instance of a job executing.
+  - shell: bash
+    run: echo 'PREFIX=${{ github.workflow }}-${{ github.job }}-${{ runner.os }}-${{ inputs.go-version }}-matrix(${{ join(matrix.*,'|') }})' >> $GITHUB_ENV
+
+  # Cache the Go Modules downloaded during the job.
+  - uses: actions/cache@v2
+    with:
+      path: ~/go/pkg/mod
+      key: ${{ env.PREFIX }}-go-mod-${{ hashFiles('**/go.sum') }}
+      restore-keys: ${{ env.PREFIX }}-go-mod
+
+  # Cache any build and test artifacts during the job, which will speed up
+  # rebuilds and cause test runs to skip tests that have no reason to rerun.
+  # Don't run this for protected branches like master/main.
+  - uses: actions/cache@v2
+    if: ${{ github.ref_protected || github.ref != 'master' || github.ref != 'main' }}
+    with:
+      path: ~/.cache/go-build
+      key: ${{ env.PREFIX }}-go-build-${{ github.ref }}-${{ hashFiles('**', '!.git') }}
+      restore-keys: |
+        ${{ env.PREFIX }}-go-build-${{ github.ref }}
+        ${{ env.PREFIX }}-go-build

.github/workflows/go.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Go
-      uses: actions/setup-go@v2
+    - uses: ./.github/actions/setup-go
       with:
         go-version: ${{ matrix.go }}
-        stable: ${{ !contains(matrix.go, 'rc') }}
     - run: ./gomod.sh
     - run: ./gofmt.sh
     - run: ./gogenerate.sh
@@ -34,11 +32,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Go
-      uses: actions/setup-go@v2
+    - uses: ./.github/actions/setup-go
       with:
         go-version: ${{ matrix.go }}
-        stable: ${{ !contains(matrix.go, 'rc') }}
     - run: go build ./...
 
   test:
@@ -81,9 +77,11 @@ jobs:
       REDIS_PORT: 6379
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Go
-      uses: actions/setup-go@v2
+      with:
+        # We need to full history for git-restore-mtime to know what modification dates to use.
+        # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
+        fetch-depth: '0'
+    - uses: ./.github/actions/setup-go
       with:
         go-version: ${{ matrix.go }}
-        stable: ${{ !contains(matrix.go, 'rc') }}
     - run: go test -race -cover ./...

.github/workflows/horizon.yml

@@ -14,6 +14,7 @@ jobs:
         go: [1.17]
         pg: [9.6.5]
         ingestion-backend: [db, captive-core, captive-core-remote-storage]
+        captive-core: [18.0.3-746.f3baea6.focal]
     runs-on: ${{ matrix.os }}
     services:
       postgres:
@@ -38,20 +39,21 @@ jobs:
       PGDATABASE: postgres
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Go
-      uses: actions/setup-go@v2
+      with:
+        # We need to full history for git-restore-mtime to know what modification dates to use.
+        # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
+        fetch-depth: '0'
+    - uses: ./.github/actions/setup-go
       with:
         go-version: ${{ matrix.go }}
     - name: Pull latest Stellar Core image
       run: docker pull stellar/stellar-core
     - if: ${{ startsWith(matrix.ingestion-backend, 'captive-core') }}
       name: Install and enable Captive Core
-      env:
-        CAPTIVE_CORE_VERSION: 18.0.3-746.f3baea6.focal
       run: |
         sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
         sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
-        sudo apt-get update && sudo apt-get install -y stellar-core=$CAPTIVE_CORE_VERSION
+        sudo apt-get update && sudo apt-get install -y stellar-core=${{ matrix.captive-core }}
         echo "Using stellar core version $(stellar-core version)"
         echo 'HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=true' >> $GITHUB_ENV
         echo 'CAPTIVE_CORE_BIN=/usr/bin/stellar-core'       >> $GITHUB_ENV

services/horizon/internal/integration/db_test.go

@@ -3,6 +3,7 @@ package integration
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strconv"
 	"testing"
@@ -538,6 +539,9 @@ func command(horizonConfig horizon.Config, args ...string) []string {
 		// due to ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING
 		"--checkpoint-frequency",
 		"8",
+		// Create the storage directory outside of the source repo,
+		// otherwise it will break Golang test caching.
+		"--captive-core-storage-path=" + os.TempDir(),
 	}, args...)
 }
 

services/horizon/internal/test/integration/integration.go

@@ -309,10 +309,13 @@ func (i *Test) StartHorizon() error {
 			hostname,
 			stellarCorePostgresPort,
 		),
-		"stellar-core-binary-path":      coreBinaryPath,
-		"captive-core-config-path":      captiveCoreConfigPath,
-		"captive-core-http-port":        "21626",
-		"captive-core-use-db":           captiveCoreUseDB,
+		"stellar-core-binary-path": coreBinaryPath,
+		"captive-core-config-path": captiveCoreConfigPath,
+		"captive-core-http-port":   "21626",
+		"captive-core-use-db":      captiveCoreUseDB,
+		// Create the storage directory outside of the source repo,
+		// otherwise it will break Golang test caching.
+		"captive-core-storage-path":     os.TempDir(),
 		"enable-captive-core-ingestion": strconv.FormatBool(len(coreBinaryPath) > 0),
 		"ingest":                        "true",
 		"history-archive-urls":          fmt.Sprintf("http://%s:%d", hostname, historyArchivePort),