diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000000..3e0eea18cb --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,23 @@ +# Github Workflows + +The execution order of the workflows is as follows: + +1. [Linting](golangci-lint.yml) +2. Main pipeline, will run in parallel: + - [Regular pipeline](ci.yml) + - [Reaper Off pipeline](ci-reaper-off.yml) +3. [Modules generator pipeline](ci-modulegen.yml) +4. Modules pipeline, will run in parallel: + - All modules' pipelines + - All examples' pipelines + +_MermaidJS diagram created with https://mermaid.live._ + +```mermaid +flowchart TD + A[golangci-lint] -->|Reaper-on| B(Main Pipeline) + A[golangci-lint] -->|Reaper-off| B(Main Pipeline) + B -->|Module Generator| C[Module generator] + C -->|Modules| D[All modules] + C -->|Examples| E[All examples] +``` diff --git a/.github/workflows/ci-modulegen.yml b/.github/workflows/ci-modulegen.yml new file mode 100644 index 0000000000..64a7442d51 --- /dev/null +++ b/.github/workflows/ci-modulegen.yml @@ -0,0 +1,59 @@ +name: Module generator pipeline + +on: + workflow_run: + workflows: [golangci-lint / static-analysis] + types: + - completed + push: + paths-ignore: + - 'mkdocs.yml' + - 'docs/**' + - 'README.md' + pull_request: + paths-ignore: + - 'mkdocs.yml' + - 'docs/**' + - 'README.md' + +concurrency: + group: "${{ github.workflow }}-${{ github.head_ref || github.sha }}" + cancel-in-progress: true + +jobs: + test: + strategy: + matrix: + go-version: [1.19.x, 1.x] + platform: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.platform }} + env: + TESTCONTAINERS_RYUK_DISABLED: "false" + steps: + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: modVerify + working-directory: ./modulegen + run: go mod verify + + - name: ensure compilation + working-directory: ./modulegen + run: go build + + - name: Run module generator tests + working-directory: ./modulegen + run: make test-unit + + - name: Test Summary + uses: test-summary/action@4ee9ece4bca777a38f05c8fc578ac2007fe266f7 + with: + paths: "**/TEST-*.xml" + if: always() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6a08f52ed..e16af48c1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,10 @@ name: Main pipeline on: + workflow_run: + workflows: [golangci-lint / static-analysis] + types: + - completed push: paths-ignore: - 'mkdocs.yml' @@ -55,10 +59,6 @@ jobs: if: ${{ matrix.platform == 'ubuntu-latest' }} run: make test-unit - - name: Run module generator tests - if: ${{ matrix.platform == 'ubuntu-latest' }} - run: make -C modulegen test-unit - - name: Run checker run: | ./scripts/check_environment.sh diff --git a/.github/workflows/cockroachdb-example.yml b/.github/workflows/cockroachdb-example.yml index 82320fbb06..401daef5a5 100644 --- a/.github/workflows/cockroachdb-example.yml +++ b/.github/workflows/cockroachdb-example.yml @@ -1,6 +1,10 @@ name: Cockroachdb example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/consul-example.yml b/.github/workflows/consul-example.yml index 513fe1a218..5e33d65ed5 100644 --- a/.github/workflows/consul-example.yml +++ b/.github/workflows/consul-example.yml @@ -1,6 +1,10 @@ name: Consul example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/datastore-example.yml b/.github/workflows/datastore-example.yml index 9876c2a968..617f4ddd2b 100644 --- a/.github/workflows/datastore-example.yml +++ b/.github/workflows/datastore-example.yml @@ -1,6 +1,10 @@ name: Datastore example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/firestore-example.yml b/.github/workflows/firestore-example.yml index 547167220d..3ced6c0482 100644 --- a/.github/workflows/firestore-example.yml +++ b/.github/workflows/firestore-example.yml @@ -1,6 +1,10 @@ name: Firestore example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-compose.yml b/.github/workflows/module-compose.yml index 9d385a1950..3647d3166c 100644 --- a/.github/workflows/module-compose.yml +++ b/.github/workflows/module-compose.yml @@ -1,6 +1,10 @@ name: Compose module pipeline on: + workflow_run: + workflows: [Main pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-couchbase.yml b/.github/workflows/module-couchbase.yml index 5924370f9b..caba2fa15e 100644 --- a/.github/workflows/module-couchbase.yml +++ b/.github/workflows/module-couchbase.yml @@ -1,6 +1,10 @@ name: Couchbase module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-localstack.yml b/.github/workflows/module-localstack.yml index 8c7d4924d5..9cf584f2a0 100644 --- a/.github/workflows/module-localstack.yml +++ b/.github/workflows/module-localstack.yml @@ -1,6 +1,10 @@ name: LocalStack module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-mysql.yml b/.github/workflows/module-mysql.yml index 6b79298f77..02444ded88 100644 --- a/.github/workflows/module-mysql.yml +++ b/.github/workflows/module-mysql.yml @@ -1,6 +1,10 @@ name: MySQL module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-neo4j.yml b/.github/workflows/module-neo4j.yml index 07f4f55727..97fe5bc826 100644 --- a/.github/workflows/module-neo4j.yml +++ b/.github/workflows/module-neo4j.yml @@ -1,6 +1,10 @@ name: Neo4j module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-postgres.yml b/.github/workflows/module-postgres.yml index 0d26b5322e..7bdddf18f2 100644 --- a/.github/workflows/module-postgres.yml +++ b/.github/workflows/module-postgres.yml @@ -1,6 +1,10 @@ name: Postgres module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-pulsar.yml b/.github/workflows/module-pulsar.yml index c4c4f6234c..265a5773fb 100644 --- a/.github/workflows/module-pulsar.yml +++ b/.github/workflows/module-pulsar.yml @@ -1,6 +1,10 @@ name: Pulsar module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-redis.yml b/.github/workflows/module-redis.yml index 09da677dc4..dd45a3cab5 100644 --- a/.github/workflows/module-redis.yml +++ b/.github/workflows/module-redis.yml @@ -1,6 +1,10 @@ name: Redis module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/module-vault.yml b/.github/workflows/module-vault.yml index 4ad758e8ef..c75010cc51 100644 --- a/.github/workflows/module-vault.yml +++ b/.github/workflows/module-vault.yml @@ -1,6 +1,10 @@ name: vault module pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/mongodb-example.yml b/.github/workflows/mongodb-example.yml index 9dd988c709..7990d2ab3f 100644 --- a/.github/workflows/mongodb-example.yml +++ b/.github/workflows/mongodb-example.yml @@ -1,6 +1,10 @@ name: MongoDB example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/nginx-example.yml b/.github/workflows/nginx-example.yml index 498075c371..15685fe9d0 100644 --- a/.github/workflows/nginx-example.yml +++ b/.github/workflows/nginx-example.yml @@ -1,6 +1,10 @@ name: Nginx example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/pubsub-example.yml b/.github/workflows/pubsub-example.yml index 3dc278fbcd..0d914fbd6f 100644 --- a/.github/workflows/pubsub-example.yml +++ b/.github/workflows/pubsub-example.yml @@ -1,6 +1,10 @@ name: Pubsub example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/spanner-example.yml b/.github/workflows/spanner-example.yml index 27fcabdb43..fd0ed2021b 100644 --- a/.github/workflows/spanner-example.yml +++ b/.github/workflows/spanner-example.yml @@ -1,6 +1,10 @@ name: Spanner example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/.github/workflows/toxiproxy-example.yml b/.github/workflows/toxiproxy-example.yml index c8383f38db..4a6eabbfad 100644 --- a/.github/workflows/toxiproxy-example.yml +++ b/.github/workflows/toxiproxy-example.yml @@ -1,6 +1,10 @@ name: Toxiproxy example pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/modulegen/_template/ci.yml.tmpl b/modulegen/_template/ci.yml.tmpl index dd5f82d137..7c2c8a8f89 100644 --- a/modulegen/_template/ci.yml.tmpl +++ b/modulegen/_template/ci.yml.tmpl @@ -1,6 +1,10 @@ {{ $lower := ToLower }}name: {{ Title }} {{ ExampleType }} pipeline on: + workflow_run: + workflows: [Module generator pipeline] + types: + - completed push: paths-ignore: - 'mkdocs.yml' diff --git a/modulegen/main_test.go b/modulegen/main_test.go index d49651f660..bd8f2ca14b 100644 --- a/modulegen/main_test.go +++ b/modulegen/main_test.go @@ -411,7 +411,7 @@ func assertExampleDocContent(t *testing.T, example Example, exampleDocFile strin lower := example.Lower() title := example.Title() - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[0], "# "+title) assert.Equal(t, data[2], `Not available until the next release of testcontainers-go :material-tag: main`) assert.Equal(t, data[4], "## Introduction") @@ -435,7 +435,7 @@ func assertExampleTestContent(t *testing.T, example Example, exampleTestFile str content, err := os.ReadFile(exampleTestFile) assert.Nil(t, err) - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[0], "package "+example.Lower()) assert.Equal(t, data[7], "func Test"+example.Title()+"(t *testing.T) {") assert.Equal(t, data[10], "\tcontainer, err := RunContainer(ctx)") @@ -451,7 +451,7 @@ func assertExampleContent(t *testing.T, example Example, exampleFile string) { exampleName := example.Title() entrypoint := example.Entrypoint() - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[0], "package "+lower) assert.Equal(t, data[8], "// "+containerName+" represents the "+exampleName+" container type used in the module") assert.Equal(t, data[9], "type "+containerName+" struct {") @@ -469,14 +469,14 @@ func assertExampleGithubWorkflowContent(t *testing.T, example Example, exampleWo lower := example.Lower() title := example.Title() - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, "name: "+title+" "+example.Type()+" pipeline", data[0]) - assert.Equal(t, " test-"+lower+":", data[19]) - assert.Equal(t, " go-version: ${{ matrix.go-version }}", data[29]) - assert.Equal(t, " working-directory: ./"+example.ParentDir()+"/"+lower, data[36]) + assert.Equal(t, " test-"+lower+":", data[23]) + assert.Equal(t, " go-version: ${{ matrix.go-version }}", data[33]) assert.Equal(t, " working-directory: ./"+example.ParentDir()+"/"+lower, data[40]) assert.Equal(t, " working-directory: ./"+example.ParentDir()+"/"+lower, data[44]) - assert.Equal(t, " paths: \"**/TEST-"+lower+"*.xml\"", data[54]) + assert.Equal(t, " working-directory: ./"+example.ParentDir()+"/"+lower, data[48]) + assert.Equal(t, " paths: \"**/TEST-"+lower+"*.xml\"", data[58]) } // assert content go.mod @@ -484,7 +484,7 @@ func assertGoModContent(t *testing.T, example Example, goModFile string) { content, err := os.ReadFile(goModFile) assert.Nil(t, err) - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, "module github.com/testcontainers/testcontainers-go/"+example.ParentDir()+"/"+example.Lower(), data[0]) assert.Equal(t, "\tgithub.com/testcontainers/testcontainers-go "+example.TCVersion, data[5]) } @@ -494,7 +494,7 @@ func assertMakefileContent(t *testing.T, example Example, makefile string) { content, err := os.ReadFile(makefile) assert.Nil(t, err) - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[4], "\t$(MAKE) test-"+example.Lower()) } @@ -534,7 +534,23 @@ func assertToolsGoContent(t *testing.T, example Example, tools string) { content, err := os.ReadFile(tools) assert.Nil(t, err) - data := strings.Split(string(content), "\n") + data := strings.Split(sanitiseContent(string(content)), "\n") assert.Equal(t, data[3], "// This package contains the tool dependencies of the "+example.Title()+" "+example.Type()+".") assert.Equal(t, data[5], "package tools") } + +// sanitiseContent removes the carriage return from a string +// This is needed because on Windows, we need to remove the carriage return while asserting the content +func sanitiseContent(s string) string { + fns := []func(string) string{ + func(s string) string { + return strings.ReplaceAll(s, "\r", "") + }, + } + + for _, fn := range fns { + s = fn(s) + } + + return s +}