From 381c2c5c994da3b197b4d38c1fa6c57c34cb6a3d Mon Sep 17 00:00:00 2001 From: Sunjay Bhatia <5337253+sunjayBhatia@users.noreply.github.com> Date: Tue, 24 Oct 2023 16:01:18 -0400 Subject: [PATCH] Run Gateway API conformance and generate report on tagged builds (#5893) Signed-off-by: Sunjay Bhatia --- .github/workflows/build_tag.yaml | 42 ++++++++++++ .gitignore | 3 + .../gatewayapi/gateway_conformance_test.go | 65 +++++++++++++++++-- test/scripts/run-gateway-conformance.sh | 5 ++ 4 files changed, 111 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_tag.yaml b/.github/workflows/build_tag.yaml index 7c1f3f56fb7..165b54cdc72 100644 --- a/.github/workflows/build_tag.yaml +++ b/.github/workflows/build_tag.yaml @@ -13,7 +13,9 @@ on: - 'v[0-9]*.[0-9]*.[0-9]*alpha*' - 'v[0-9]*.[0-9]*.[0-9]*rc*' env: + GOPROXY: https://proxy.golang.org/ SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + GO_VERSION: 1.21.3 jobs: build: runs-on: ubuntu-latest @@ -41,3 +43,43 @@ jobs: steps: ${{ toJson(steps) }} channel: '#contour-ci-notifications' if: failure() + gateway-conformance-report: + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v4 + - uses: actions/cache@v3 + with: + # * Module download cache + # * Build cache (Linux) + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-${{ github.job }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ github.job }}-go- + - uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + - name: add deps to path + run: | + ./hack/actions/install-kubernetes-toolchain.sh $GITHUB_WORKSPACE/bin + echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH + - name: Gateway API conformance tests + env: + GENERATE_GATEWAY_CONFORMANCE_REPORT: "true" + run: | + export CONTOUR_E2E_IMAGE="ghcr.io/projectcontour/contour:$(git describe --tags)" + make gateway-conformance + - name: Upload gateway conformance report + uses: actions/upload-artifact@v3 + with: + name: gateway-conformance-report + path: gateway-conformance-report/projectcontour-contour-*.yaml + - uses: act10ns/slack@v2 + with: + status: ${{ job.status }} + steps: ${{ toJson(steps) }} + channel: '#contour-ci-notifications' + if: ${{ failure() && github.ref == 'refs/heads/main' }} diff --git a/.gitignore b/.gitignore index 8f3ca74f552..67eadd05f1c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,8 @@ site/public # Saved container image archive image/ +# Gateway API conformance report output +gateway-conformance-report/ + # Compiled tests from Ginkgo *.test diff --git a/test/conformance/gatewayapi/gateway_conformance_test.go b/test/conformance/gatewayapi/gateway_conformance_test.go index 7247eb22f9b..3afc8c97fe1 100644 --- a/test/conformance/gatewayapi/gateway_conformance_test.go +++ b/test/conformance/gatewayapi/gateway_conformance_test.go @@ -16,9 +16,14 @@ package gatewayapi import ( + "fmt" + "os" + "path/filepath" "testing" + "time" "github.com/bombsimon/logrusr/v4" + "github.com/distribution/reference" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/sets" @@ -28,9 +33,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" + conformance_v1alpha1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" "sigs.k8s.io/gateway-api/conformance/tests" "sigs.k8s.io/gateway-api/conformance/utils/flags" "sigs.k8s.io/gateway-api/conformance/utils/suite" + "sigs.k8s.io/yaml" ) func TestGatewayConformance(t *testing.T) { @@ -48,7 +55,7 @@ func TestGatewayConformance(t *testing.T) { require.NoError(t, v1alpha2.AddToScheme(client.Scheme())) require.NoError(t, v1beta1.AddToScheme(client.Scheme())) - cSuite := suite.New(suite.Options{ + cSuiteOptions := suite.Options{ Client: client, // This clientset is needed in addition to the client only because // controller-runtime client doesn't support non CRUD sub-resources yet (https://github.com/kubernetes-sigs/controller-runtime/issues/452). @@ -68,8 +75,58 @@ func TestGatewayConformance(t *testing.T) { ExemptFeatures: sets.New( suite.SupportMesh, ), - }) - cSuite.Setup(t) - cSuite.Run(t, tests.ConformanceTests) + } + if os.Getenv("GENERATE_GATEWAY_CONFORMANCE_REPORT") == "true" { + reportDir, ok := os.LookupEnv("GATEWAY_CONFORMANCE_REPORT_OUTDIR") + require.True(t, ok, "GATEWAY_CONFORMANCE_REPORT_OUTDIR not set") + image, ok := os.LookupEnv("CONTOUR_E2E_IMAGE") + require.True(t, ok, "CONTOUR_E2E_IMAGE not set") + + imageRef, err := reference.Parse(image) + require.NoErrorf(t, err, "CONTOUR_E2E_IMAGE invalid image ref: %s", imageRef) + taggedImage, ok := imageRef.(reference.NamedTagged) + require.True(t, ok) + require.NotEmpty(t, taggedImage) + + // Workaround since the experimental suite doesn't properly + // exclude tests we don't want to run using the ExemptFeatures + // field. + cSuiteOptions.EnableAllSupportedFeatures = false + cSuiteOptions.SupportedFeatures = suite.AllFeatures.Delete(suite.MeshCoreFeatures.UnsortedList()...) + + cSuite, err := suite.NewExperimentalConformanceTestSuite(suite.ExperimentalConformanceOptions{ + Options: cSuiteOptions, + Implementation: conformance_v1alpha1.Implementation{ + Organization: "projectcontour", + Project: "contour", + URL: "https://projectcontour.io/", + Version: taggedImage.Tag(), + Contact: []string{"@projectcontour/maintainers"}, + }, + ConformanceProfiles: sets.New[suite.ConformanceProfileName]( + suite.HTTPConformanceProfileName, + suite.TLSConformanceProfileName, + ), + }) + require.NoError(t, err) + + cSuite.Setup(t) + require.NoError(t, cSuite.Run(t, tests.ConformanceTests)) + + report, err := cSuite.Report() + require.NoError(t, err, "failed generating conformance report") + rawReport, err := yaml.Marshal(report) + require.NoError(t, err) + t.Logf("Conformance report:\n%s", string(rawReport)) + + require.NoError(t, os.MkdirAll(reportDir, 0o755)) + outFile := filepath.Join(reportDir, fmt.Sprintf("projectcontour-contour-%d.yaml", time.Now().UnixNano())) + require.NoError(t, os.WriteFile(outFile, rawReport, 0o600)) + t.Logf("Report written to: %s", outFile) + } else { + cSuite := suite.New(cSuiteOptions) + cSuite.Setup(t) + cSuite.Run(t, tests.ConformanceTests) + } } diff --git a/test/scripts/run-gateway-conformance.sh b/test/scripts/run-gateway-conformance.sh index 4c86bc05d2b..de086ca99c9 100755 --- a/test/scripts/run-gateway-conformance.sh +++ b/test/scripts/run-gateway-conformance.sh @@ -18,8 +18,13 @@ set -o pipefail set -o errexit set -o nounset +readonly HERE=$(cd "$(dirname "$0")" && pwd) +readonly REPO=$(cd "${HERE}/../.." && pwd) + readonly KUBECTL=${KUBECTL:-kubectl} export CONTOUR_IMG=${CONTOUR_E2E_IMAGE:-ghcr.io/projectcontour/contour:main} +export GENERATE_GATEWAY_CONFORMANCE_REPORT=${GENERATE_GATEWAY_CONFORMANCE_REPORT:-false} +export GATEWAY_CONFORMANCE_REPORT_OUTDIR="${REPO}/gateway-conformance-report" echo "Using Contour image: ${CONTOUR_IMG}" echo "Using Gateway API version: ${GATEWAY_API_VERSION}"