From 90f2a3e561e4d5a8220141171545cbae0882b2d5 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 23 Mar 2023 18:25:59 +0800 Subject: [PATCH 01/13] test: generate coverage profile during e2e tests Signed-off-by: Billy Zha --- .github/workflows/build.yml | 1 + test/e2e/internal/utils/init.go | 28 +++++++++++++++++++++++++++- test/e2e/scripts/e2e.sh | 26 ++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f7bb92c74..33860e0fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,6 +46,7 @@ jobs: sh $GITHUB_WORKSPACE/test/e2e/scripts/e2e.sh $GITHUB_WORKSPACE --clean env: ORAS_PATH: bin/linux/amd64/oras + COVERAGE_DUMP_ROOT: .cover - name: Check Version run: bin/linux/amd64/oras version - name: Upload coverage to codecov.io diff --git a/test/e2e/internal/utils/init.go b/test/e2e/internal/utils/init.go index f0922c0d8..33018f2d7 100644 --- a/test/e2e/internal/utils/init.go +++ b/test/e2e/internal/utils/init.go @@ -36,6 +36,9 @@ var Host string // FallbackHost points to the registry service where fallback E2E specs will be run against. var FallbackHost string +// Path to generate the coverage report. +var CovDumpPath string + func init() { Host = os.Getenv(RegHostKey) if Host == "" { @@ -72,6 +75,25 @@ func init() { } BeforeSuite(func() { ORASPath = os.Getenv("ORAS_PATH") + if covDumpRoot := os.Getenv("COVERAGE_DUMP_ROOT"); covDumpRoot != "" { + if ORASPath != "" { + fmt.Printf("Pre-built oras ignored: %s", ORASPath) + ORASPath = "" + } + if filepath.IsAbs(covDumpRoot) { + CovDumpPath = covDumpRoot + } else if workspacePath := os.Getenv("GITHUB_WORKSPACE"); workspacePath != "" { + CovDumpPath = filepath.Join(workspacePath, "test/e2e", covDumpRoot) + } else { + // local debugging + CovDumpPath = filepath.Join(pwd, "..", "..", covDumpRoot) + } + + // confirm the existence of dump folder + err := os.MkdirAll(CovDumpPath, 0700) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + fmt.Printf("Coverage file dump path: %q\n", CovDumpPath) + } if filepath.IsAbs(ORASPath) { fmt.Printf("Testing based on pre-built binary locates in %q\n", ORASPath) } else if workspacePath := os.Getenv("GITHUB_WORKSPACE"); ORASPath != "" && workspacePath != "" { @@ -82,7 +104,11 @@ func init() { fmt.Printf("Testing based on pre-built binary locates in %q\n", ORASPath) } else { // fallback to native build to facilitate local debugging - ORASPath, err = gexec.Build("oras.land/oras/cmd/oras") + buildArgs := []string{} + if CovDumpPath != "" { + buildArgs = append(buildArgs, "-coverpkg", "oras.land/oras/cmd/oras/...,oras.land/oras/internal/...") + } + ORASPath, err = gexec.Build("oras.land/oras/cmd/oras", buildArgs...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) DeferCleanup(gexec.CleanupBuildArtifacts) fmt.Printf("Testing based on temp binary locates in %q\n", ORASPath) diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index f434cfc92..9cebaf17a 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -43,25 +43,43 @@ fi oras_container_name="oras-e2e" upstream_container_name="oras-e2e-fallback" +e2e_root="${repo_root}/test/e2e" echo " === preparing oras distribution === " run_registry \ - ${repo_root}/test/e2e/testdata/distribution/mount \ + ${e2e_root}/testdata/distribution/mount \ ghcr.io/oras-project/registry:v1.0.0-rc.4 \ $oras_container_name \ $ORAS_REGISTRY_PORT echo " === preparing upstream distribution === " run_registry \ - ${repo_root}/test/e2e/testdata/distribution/mount_fallback \ + ${e2e_root}/testdata/distribution/mount_fallback \ registry:2.8.1 \ $upstream_container_name \ $ORAS_REGISTRY_FALLBACK_PORT +if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then + rm ${e2e_root}/${COVERAGE_DUMP_ROOT} -rf +fi + +echo " === setup coverage instrumenting == " +if [[ ($GITHUB_REF_NAME == release-* && $GITHUB_REF_TYPE == branch) || ($GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag) ]]; then + unset COVERAGE_DUMP_ROOT +fi + echo " === run tests === " -if ! ginkgo -r -p --succinct suite; then +ginkgo -r -p --succinct suite || fail=true + +if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then + echo " === generating code cov report === " + go tool covdata textfmt -i="${e2e_root}/${COVERAGE_DUMP_ROOT}" -o ${e2e_root}/coverage.txt || true +fi + +if [ "${fail}" = 'true' ]; then + echo " === retriving registry error logs === " echo '-------- oras distribution trace -------------' docker logs -t --tail 200 $oras_container_name echo '-------- upstream distribution trace -------------' docker logs -t --tail 200 $upstream_container_name exit 1 -fi +f From ec3d6c445cbf88bf85991ae591e8b73b4565a224 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 23 Mar 2023 18:28:33 +0800 Subject: [PATCH 02/13] code clean Signed-off-by: Billy Zha --- test/e2e/internal/utils/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/internal/utils/init.go b/test/e2e/internal/utils/init.go index 33018f2d7..1928d83d7 100644 --- a/test/e2e/internal/utils/init.go +++ b/test/e2e/internal/utils/init.go @@ -77,7 +77,7 @@ func init() { ORASPath = os.Getenv("ORAS_PATH") if covDumpRoot := os.Getenv("COVERAGE_DUMP_ROOT"); covDumpRoot != "" { if ORASPath != "" { - fmt.Printf("Pre-built oras ignored: %s", ORASPath) + fmt.Printf("Pre-built oras ignored: %s\n", ORASPath) ORASPath = "" } if filepath.IsAbs(covDumpRoot) { From 2f54924859dc4b8f8bbc5b1b2ac057ed83f819d5 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 23 Mar 2023 18:54:00 +0800 Subject: [PATCH 03/13] fix script Signed-off-by: Billy Zha --- test/e2e/scripts/e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index 9cebaf17a..7018ca65f 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -63,7 +63,7 @@ if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then fi echo " === setup coverage instrumenting == " -if [[ ($GITHUB_REF_NAME == release-* && $GITHUB_REF_TYPE == branch) || ($GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag) ]]; then +if [[ $GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag ]]; then unset COVERAGE_DUMP_ROOT fi From 41b109f57af1a6ebd4a78713e415b3ed19786799 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 23 Mar 2023 19:06:57 +0800 Subject: [PATCH 04/13] fix e2e Signed-off-by: Billy Zha --- test/e2e/internal/utils/exec.go | 1 + test/e2e/suite/command/blob.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/internal/utils/exec.go b/test/e2e/internal/utils/exec.go index b827ba019..a998828da 100644 --- a/test/e2e/internal/utils/exec.go +++ b/test/e2e/internal/utils/exec.go @@ -189,6 +189,7 @@ func (opts *ExecOption) Exec() *gexec.Session { opts.binary = ORASPath } cmd = exec.Command(opts.binary, opts.args...) + cmd.Env = append(os.Environ(), fmt.Sprintf("GOCOVERDIR=%s", CovDumpPath)) cmd.Stdin = opts.stdin if opts.workDir != "" { // switch working directory diff --git a/test/e2e/suite/command/blob.go b/test/e2e/suite/command/blob.go index 15f13f408..12ca81dd2 100644 --- a/test/e2e/suite/command/blob.go +++ b/test/e2e/suite/command/blob.go @@ -41,14 +41,14 @@ var _ = Describe("ORAS beginners:", func() { repo := fmt.Sprintf(repoFmt, "push", "password-stdin") ORAS("blob", "push", RegistryRef(Host, repo, ""), "--password-stdin", "-"). ExpectFailure(). - MatchTrimmedContent("Error: `-` read file from input and `--password-stdin` read password from input cannot be both used").Exec() + MatchErrKeyWords("Error: `-` read file from input and `--password-stdin` read password from input cannot be both used").Exec() }) It("should fail to push a blob from stdin but no blob size provided", func() { repo := fmt.Sprintf(repoFmt, "push", "no-size") ORAS("blob", "push", RegistryRef(Host, repo, pushDigest), "-"). WithInput(strings.NewReader(pushContent)). ExpectFailure(). - MatchTrimmedContent("Error: `--size` must be provided if the blob is read from stdin").Exec() + MatchErrKeyWords("Error: `--size` must be provided if the blob is read from stdin").Exec() }) It("should fail to push a blob from stdin if invalid blob size provided", func() { From 4cc4279dba1f5e43addcf4917aa41e18a4397e08 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 23 Mar 2023 19:11:26 +0800 Subject: [PATCH 05/13] fix script Signed-off-by: Billy Zha --- test/e2e/scripts/e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index 7018ca65f..22ad354dc 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -82,4 +82,4 @@ if [ "${fail}" = 'true' ]; then echo '-------- upstream distribution trace -------------' docker logs -t --tail 200 $upstream_container_name exit 1 -f +if From 5f7a6749e2f888e3159c80a2c2302f2c50f8583e Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 23 Mar 2023 19:16:09 +0800 Subject: [PATCH 06/13] fix script Signed-off-by: Billy Zha --- test/e2e/scripts/e2e.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index 22ad354dc..a1ba508a3 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -58,15 +58,15 @@ run_registry \ $upstream_container_name \ $ORAS_REGISTRY_FALLBACK_PORT -if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then - rm ${e2e_root}/${COVERAGE_DUMP_ROOT} -rf -fi - echo " === setup coverage instrumenting == " if [[ $GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag ]]; then unset COVERAGE_DUMP_ROOT fi +if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then + rm ${e2e_root}/${COVERAGE_DUMP_ROOT} -rf +fi + echo " === run tests === " ginkgo -r -p --succinct suite || fail=true @@ -82,4 +82,4 @@ if [ "${fail}" = 'true' ]; then echo '-------- upstream distribution trace -------------' docker logs -t --tail 200 $upstream_container_name exit 1 -if +fi \ No newline at end of file From f49663d502465f2ab98316ce84a76632120d69d4 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 28 Mar 2023 11:45:27 +0000 Subject: [PATCH 07/13] use makefile Signed-off-by: Billy Zha --- .gitignore | 1 + Makefile | 4 ++++ test/e2e/scripts/e2e.sh | 4 +++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7b8e4c7e0..ee4ccdf31 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ debug # Custom coverage.txt +test/e2e/coverage.txt bin/ dist/ *.tar.gz diff --git a/Makefile b/Makefile index 6957c0d93..1a64136e9 100644 --- a/Makefile +++ b/Makefile @@ -124,3 +124,7 @@ sign: for f in $$(ls _dist/*.{gz,txt} 2>/dev/null) ; do \ gpg --armor --detach-sign $${f} ; \ done + +.PHONY: e2e-covdata +e2e-covdata: + $(GO_EXE) tool covdata textfmt -i="test/e2e/${COVERAGE_DUMP_ROOT}" -o test/e2e/coverage.txt \ No newline at end of file diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index a1ba508a3..77d84e472 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -60,8 +60,10 @@ run_registry \ echo " === setup coverage instrumenting == " if [[ $GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag ]]; then + echo "coverage instrumentation skipped" unset COVERAGE_DUMP_ROOT fi +return if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then rm ${e2e_root}/${COVERAGE_DUMP_ROOT} -rf @@ -72,7 +74,7 @@ ginkgo -r -p --succinct suite || fail=true if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then echo " === generating code cov report === " - go tool covdata textfmt -i="${e2e_root}/${COVERAGE_DUMP_ROOT}" -o ${e2e_root}/coverage.txt || true + make -C ${repo_root} e2e-covdata || true fi if [ "${fail}" = 'true' ]; then From 9776da4c8617551c53ba1c11d6fb8de3f047a64f Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 28 Mar 2023 11:47:45 +0000 Subject: [PATCH 08/13] code clean Signed-off-by: Billy Zha --- test/e2e/scripts/e2e.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index 77d84e472..3b3624ffb 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -63,7 +63,6 @@ if [[ $GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag ]]; then echo "coverage instrumentation skipped" unset COVERAGE_DUMP_ROOT fi -return if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then rm ${e2e_root}/${COVERAGE_DUMP_ROOT} -rf From bc9aa9327e999bb5dcd35e7df13ebb7377704acb Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 28 Mar 2023 11:59:07 +0000 Subject: [PATCH 09/13] update tests and gitignore Signed-off-by: Billy Zha --- .github/workflows/build.yml | 2 +- .gitignore | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33860e0fe..4a2883bd9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: run: make test - name: Run E2E Tests run: | - sh $GITHUB_WORKSPACE/test/e2e/scripts/e2e.sh $GITHUB_WORKSPACE --clean + bash $GITHUB_WORKSPACE/test/e2e/scripts/e2e.sh $GITHUB_WORKSPACE --clean env: ORAS_PATH: bin/linux/amd64/oras COVERAGE_DUMP_ROOT: .cover diff --git a/.gitignore b/.gitignore index ee4ccdf31..0c2f8f30d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ debug # Custom coverage.txt test/e2e/coverage.txt +**/covcounters.* +**/covmeta.* bin/ dist/ *.tar.gz From d3ef9c1ae6755368f968c522058ebbfa0555bc06 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 23 Mar 2023 18:25:59 +0800 Subject: [PATCH 10/13] test: generate coverage profile during e2e tests Signed-off-by: Billy Zha --- test/e2e/scripts/e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index 3b3624ffb..b0a4f0747 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -83,4 +83,4 @@ if [ "${fail}" = 'true' ]; then echo '-------- upstream distribution trace -------------' docker logs -t --tail 200 $upstream_container_name exit 1 -fi \ No newline at end of file +fi From 71175ed323c0e2288787edebcd22fc4318697745 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 28 Mar 2023 11:45:27 +0000 Subject: [PATCH 11/13] use makefile Signed-off-by: Billy Zha --- test/e2e/scripts/e2e.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index b0a4f0747..66dfe80ab 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -63,6 +63,7 @@ if [[ $GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag ]]; then echo "coverage instrumentation skipped" unset COVERAGE_DUMP_ROOT fi +return if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then rm ${e2e_root}/${COVERAGE_DUMP_ROOT} -rf From 197e5593939053a1071abd0d1d1cecbf0f37b104 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 28 Mar 2023 11:47:45 +0000 Subject: [PATCH 12/13] code clean Signed-off-by: Billy Zha --- test/e2e/scripts/e2e.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/scripts/e2e.sh b/test/e2e/scripts/e2e.sh index 66dfe80ab..b0a4f0747 100755 --- a/test/e2e/scripts/e2e.sh +++ b/test/e2e/scripts/e2e.sh @@ -63,7 +63,6 @@ if [[ $GITHUB_REF_NAME == v* && $GITHUB_REF_TYPE == tag ]]; then echo "coverage instrumentation skipped" unset COVERAGE_DUMP_ROOT fi -return if ! [ -z ${COVERAGE_DUMP_ROOT} ]; then rm ${e2e_root}/${COVERAGE_DUMP_ROOT} -rf From 0ed4b9c8b1effe322f47259a70d319f42722e656 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 6 Apr 2023 09:22:53 +0000 Subject: [PATCH 13/13] add new line in Makefile Signed-off-by: Billy Zha --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1a64136e9..f93af9b07 100644 --- a/Makefile +++ b/Makefile @@ -127,4 +127,4 @@ sign: .PHONY: e2e-covdata e2e-covdata: - $(GO_EXE) tool covdata textfmt -i="test/e2e/${COVERAGE_DUMP_ROOT}" -o test/e2e/coverage.txt \ No newline at end of file + $(GO_EXE) tool covdata textfmt -i="test/e2e/${COVERAGE_DUMP_ROOT}" -o test/e2e/coverage.txt