diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f7bb92c74..4a2883bd9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,9 +43,10 @@ jobs: run: make test - name: Run E2E Tests run: | - sh $GITHUB_WORKSPACE/test/e2e/scripts/e2e.sh $GITHUB_WORKSPACE --clean + bash $GITHUB_WORKSPACE/test/e2e/scripts/e2e.sh $GITHUB_WORKSPACE --clean env: ORAS_PATH: bin/linux/amd64/oras + COVERAGE_DUMP_ROOT: .cover - name: Check Version run: bin/linux/amd64/oras version - name: Upload coverage to codecov.io diff --git a/.gitignore b/.gitignore index 7b8e4c7e0..0c2f8f30d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,9 @@ debug # Custom coverage.txt +test/e2e/coverage.txt +**/covcounters.* +**/covmeta.* bin/ dist/ *.tar.gz diff --git a/Makefile b/Makefile index 6957c0d93..f93af9b07 100644 --- a/Makefile +++ b/Makefile @@ -124,3 +124,7 @@ sign: for f in $$(ls _dist/*.{gz,txt} 2>/dev/null) ; do \ gpg --armor --detach-sign $${f} ; \ done + +.PHONY: e2e-covdata +e2e-covdata: + $(GO_EXE) tool covdata textfmt -i="test/e2e/${COVERAGE_DUMP_ROOT}" -o test/e2e/coverage.txt diff --git a/test/e2e/internal/utils/exec.go b/test/e2e/internal/utils/exec.go index b827ba019..a998828da 100644 --- a/test/e2e/internal/utils/exec.go +++ b/test/e2e/internal/utils/exec.go @@ -189,6 +189,7 @@ func (opts *ExecOption) Exec() *gexec.Session { opts.binary = ORASPath } cmd = exec.Command(opts.binary, opts.args...) + cmd.Env = append(os.Environ(), fmt.Sprintf("GOCOVERDIR=%s", CovDumpPath)) cmd.Stdin = opts.stdin if opts.workDir != "" { // switch working directory diff --git a/test/e2e/internal/utils/init.go b/test/e2e/internal/utils/init.go index f0922c0d8..1928d83d7 100644 --- a/test/e2e/internal/utils/init.go +++ b/test/e2e/internal/utils/init.go @@ -36,6 +36,9 @@ var Host string // FallbackHost points to the registry service where fallback E2E specs will be run against. var FallbackHost string +// Path to generate the coverage report. +var CovDumpPath string + func init() { Host = os.Getenv(RegHostKey) if Host == "" { @@ -72,6 +75,25 @@ func init() { } BeforeSuite(func() { ORASPath = os.Getenv("ORAS_PATH") + if covDumpRoot := os.Getenv("COVERAGE_DUMP_ROOT"); covDumpRoot != "" { + if ORASPath != "" { + fmt.Printf("Pre-built oras ignored: %s\n", ORASPath) + ORASPath = "" + } + if filepath.IsAbs(covDumpRoot) { + CovDumpPath = covDumpRoot + } else if workspacePath := os.Getenv("GITHUB_WORKSPACE"); workspacePath != "" { + CovDumpPath = filepath.Join(workspacePath, "test/e2e", covDumpRoot) + } else { + // local debugging + CovDumpPath = filepath.Join(pwd, "..", "..", covDumpRoot) + } + + // confirm the existence of dump folder + err := os.MkdirAll(CovDumpPath, 0700) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + fmt.Printf("Coverage file dump path: %q\n", CovDumpPath) + } if filepath.IsAbs(ORASPath) { fmt.Printf("Testing based on pre-built binary locates in %q\n", ORASPath) } else if workspacePath := os.Getenv("GITHUB_WORKSPACE"); ORASPath != "" && workspacePath != "" { @@ -82,7 +104,11 @@ func init() { fmt.Printf("Testing based on pre-built binary locates in %q\n", ORASPath) } else { // fallback to native build to facilitate local debugging - ORASPath, err = gexec.Build("oras.land/oras/cmd/oras") + buildArgs := []string{} + if CovDumpPath != "" { + buildArgs = append(buildArgs, "-coverpkg", "oras.land/oras/cmd/oras/...,oras.land/oras/internal/...") + } + ORASPath, err = gexec.Build("oras.land/oras/cmd/oras", buildArgs...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) DeferCleanup(gexec.CleanupBuildArtifacts) fmt.Printf("Testing based on temp binary locates in %q\n", ORASPath) diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index f434cfc92..b0a4f0747 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -43,22 +43,41 @@ fi oras_container_name="oras-e2e" upstream_container_name="oras-e2e-fallback" +e2e_root="${repo_root}/test/e2e" echo " === preparing oras distribution === " run_registry \ - ${repo_root}/test/e2e/testdata/distribution/mount \ + ${e2e_root}/testdata/distribution/mount \ ghcr.io/oras-project/registry:v1.0.0-rc.4 \ $oras_container_name \ $ORAS_REGISTRY_PORT echo " === preparing upstream distribution === " run_registry \ - ${repo_root}/test/e2e/testdata/distribution/mount_fallback \ + ${e2e_root}/testdata/distribution/mount_fallback \ registry:2.8.1 \ $upstream_container_name \ $ORAS_REGISTRY_FALLBACK_PORT +echo " === setup coverage instrumenting == " +if [[ $GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag ]]; then + echo "coverage instrumentation skipped" + unset COVERAGE_DUMP_ROOT +fi + +if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then + rm ${e2e_root}/${COVERAGE_DUMP_ROOT} -rf +fi + echo " === run tests === " -if ! ginkgo -r -p --succinct suite; then +ginkgo -r -p --succinct suite || fail=true + +if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then + echo " === generating code cov report === " + make -C ${repo_root} e2e-covdata || true +fi + +if [ "${fail}" = 'true' ]; then + echo " === retriving registry error logs === " echo '-------- oras distribution trace -------------' docker logs -t --tail 200 $oras_container_name echo '-------- upstream distribution trace -------------' diff --git a/test/e2e/suite/command/blob.go b/test/e2e/suite/command/blob.go index 15f13f408..12ca81dd2 100644 --- a/test/e2e/suite/command/blob.go +++ b/test/e2e/suite/command/blob.go @@ -41,14 +41,14 @@ var _ = Describe("ORAS beginners:", func() { repo := fmt.Sprintf(repoFmt, "push", "password-stdin") ORAS("blob", "push", RegistryRef(Host, repo, ""), "--password-stdin", "-"). ExpectFailure(). - MatchTrimmedContent("Error: `-` read file from input and `--password-stdin` read password from input cannot be both used").Exec() + MatchErrKeyWords("Error: `-` read file from input and `--password-stdin` read password from input cannot be both used").Exec() }) It("should fail to push a blob from stdin but no blob size provided", func() { repo := fmt.Sprintf(repoFmt, "push", "no-size") ORAS("blob", "push", RegistryRef(Host, repo, pushDigest), "-"). WithInput(strings.NewReader(pushContent)). ExpectFailure(). - MatchTrimmedContent("Error: `--size` must be provided if the blob is read from stdin").Exec() + MatchErrKeyWords("Error: `--size` must be provided if the blob is read from stdin").Exec() }) It("should fail to push a blob from stdin if invalid blob size provided", func() {