From 79167a157f3bb733b928f445009e9675c46c3baa Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Wed, 8 Jul 2020 10:59:38 -0400 Subject: [PATCH 1/4] Cirrus: Rotate keys post repo. rename Encode credentials at new repository settings page https://cirrus-ci.com/settings/repository/6707778565701632 Ref: https://cirrus-ci.org/guide/writing-tasks/#encrypted-variables Backport-of: commit 576ce0f1b501 Signed-off-by: Chris Evich Signed-off-by: Valentin Rothberg --- .cirrus.yml | 44 ++++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 96f6545384..046213b3e1 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,9 +1,9 @@ --- # Only github users with write-access can define or use encrypted variables -# This credential represents a service account with access to manage both VMs -# and storage. -gcp_credentials: ENCRYPTED[885c6e4297dd8d6f67593c42b810353af0c505a7a670e2c6fd830c56e86bbb2debcc3c18f942d0d46ab36b63521061d4] +# Ref: https://cirrus-ci.org/guide/writing-tasks/#encrypted-variables +# more specifically: https://cirrus-ci.com/settings/repository/6707778565701632 +gcp_credentials: ENCRYPTED[a28959877b2c9c36f151781b0a05407218cda646c7d047fc556e42f55e097e897ab63ee78369dae141dcf0b46a9d0cdd] # Default timeout for each task timeout_in: 120m @@ -52,6 +52,8 @@ env: BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" # Special image w/ nested-libvirt + tools for creating new cache and base images IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081" + # Name where this repositories VM images are stored + GCP_PROJECT_ID: libpod-218412 #### #### Default to NOT operating in any special-case testing mode @@ -65,18 +67,15 @@ env: #### Credentials and other secret-sauces, decrypted at runtime when authorized. #### # Freenode IRC credentials for posting status messages - IRCID: ENCRYPTED[1913f8a4572b6a6d2036232327789c4f6c0d98cde53f0336d860cd219b4cbd83863eefd93471aef8fa1079d4698e382d] - # Needed to build GCE images, within a GCE VM - SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a] - # User ID for cirrus to ssh into VMs - GCE_SSH_USERNAME: cirrus-ci - # Name where this repositories cloud resources are located - GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] - + IRCID: ENCRYPTED[0c4a3cc4ecda08bc47cd3d31592be8ae5c2bd0151bf3def00a9afd139ef1ab23a1bd0523319d076c027f9749ddb1f3c8] + # Service-account client_email - needed to build images + SERVICE_ACCOUNT: ENCRYPTED[702a8e07e27a6faf7988fcddcc068c2ef2bb182a5aa671f5ccb7fbbfb891c823aa4a7856fb17240766845dbd68bd3f90] + # Service account username part of client_email - for ssh'ing into VMs + GCE_SSH_USERNAME: ENCRYPTED[d579f2d3000bb678c9af37c3615e92bcf3726e9afc47748c129cef23ee799faaafd4baba64048329205d162069d90060] # Default VM to use unless set or modified by task gce_instance: - image_project: "libpod-218412" + image_project: $GCP_PROJECT_ID zone: "us-central1-a" # Required by Cirrus for the time being cpu: 2 memory: "4Gb" @@ -335,13 +334,6 @@ build_without_cgo_task: # Update metadata on VM images referenced by this repository state meta_task: - depends_on: - - "gating" - - "vendor" - - "varlink_api" - - "build_each_commit" - - "build_without_cgo" - container: image: "quay.io/libpod/imgts:$DEST_BRANCH" # see contrib/imgts cpu: 1 @@ -357,10 +349,10 @@ meta_task: ${IMAGE_BUILDER_CACHE_IMAGE_NAME} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_CHANGE_IN_REPO}" - GCPJSON: ENCRYPTED[950d9c64ad78f7b1f0c7e499b42dc058d2b23aa67e38b315e68f557f2aba0bf83068d4734f7b1e1bdd22deabe99629df] + GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4] # needed for output-masking purposes - GCPNAME: ENCRYPTED[b05d469a0dba8cb479cb00cc7c1f6747c91d17622fba260a986b976aa6c817d4077eacffd4613d6d5f23afc4084fab1d] - GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] + GCPNAME: ENCRYPTED[2f9738ef295a706f66a13891b40e8eaa92a89e0e87faf8bed66c41eca72bf76cfd190a6f2d0e8444c631fdf15ed32ef6] + GCPPROJECT: $GCP_PROJECT_ID timeout_in: 10m @@ -618,7 +610,7 @@ test_build_cache_images_task: auto_cancellation: $CI != "true" gce_instance: - image_project: "libpod-218412" + image_project: $GCP_PROJECT_ID zone: "us-central1-a" cpu: 4 memory: "4Gb" @@ -687,9 +679,9 @@ docs_task: depends_on: - "gating" env: - RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46] - RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1] - RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] + RELEASE_GCPJSON: ENCRYPTED[927dc01e755eaddb4242b0845cf86c9098d1e3dffac38c70aefb1487fd8b4fe6dd6ae627b3bffafaba70e2c63172664e] + RELEASE_GCPNAME: ENCRYPTED[c145e9c16b6fb88d476944a454bf4c1ccc84bb4ecaca73bdd28bdacef0dfa7959ebc8171a27b2e4064d66093b2cdba49] + RELEASE_GCPROJECT: $GCP_PROJECT_ID script: - "$SCRIPT_BASE/build_swagger.sh |& ${TIMESTAMP}" From 2c72fbdd82253835e7a082cd1ced1fa21f543bea Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Mon, 13 Jul 2020 10:34:41 -0500 Subject: [PATCH 2/4] Fix systemd pid 1 test fedora removed the systemd package from its standard container image causing our systemd pid1 test to fail. Replacing usage of fedora to ubi-init. adding ubi images to the cache for local tests. also, remove installation of test/policy.json to the system wide /etc/containers Signed-off-by: Brent Baude --- contrib/cirrus/lib.sh | 1 - test/e2e/config_amd64.go | 4 +++- test/e2e/system_df_test.go | 2 +- test/e2e/systemd_test.go | 7 +------ 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index a048a30681..c77eaca120 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -392,7 +392,6 @@ install_test_configs() { # as the default). This config prevents allocation of network address space used # by default in google cloud. https://cloud.google.com/vpc/docs/vpc#ip-ranges install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist /etc/cni/net.d/ - install -v -D -m 644 ./test/policy.json /etc/containers/ install -v -D -m 644 ./test/registries.conf /etc/containers/ } diff --git a/test/e2e/config_amd64.go b/test/e2e/config_amd64.go index 7e5765ac76..1511200589 100644 --- a/test/e2e/config_amd64.go +++ b/test/e2e/config_amd64.go @@ -5,9 +5,11 @@ var ( STORAGE_OPTIONS = "--storage-driver vfs" ROOTLESS_STORAGE_FS = "vfs" ROOTLESS_STORAGE_OPTIONS = "--storage-driver vfs" - CACHE_IMAGES = []string{ALPINE, BB, fedoraMinimal, nginx, redis, registry, infra, labels, healthcheck} + CACHE_IMAGES = []string{ALPINE, BB, fedoraMinimal, nginx, redis, registry, infra, labels, healthcheck, ubi_init, ubi_minimal} nginx = "quay.io/libpod/alpine_nginx:latest" BB_GLIBC = "docker.io/library/busybox:glibc" registry = "docker.io/library/registry:2.6" labels = "quay.io/libpod/alpine_labels:latest" + ubi_minimal = "registry.access.redhat.com/ubi8-minimal" + ubi_init = "registry.access.redhat.com/ubi8-init" ) diff --git a/test/e2e/system_df_test.go b/test/e2e/system_df_test.go index d1a43798ec..e882756f9c 100644 --- a/test/e2e/system_df_test.go +++ b/test/e2e/system_df_test.go @@ -56,7 +56,7 @@ var _ = Describe("podman system df", func() { images := strings.Fields(session.OutputToStringArray()[1]) containers := strings.Fields(session.OutputToStringArray()[2]) volumes := strings.Fields(session.OutputToStringArray()[3]) - Expect(images[1]).To(Equal("9")) + Expect(images[1]).To(Equal("11")) Expect(containers[1]).To(Equal("2")) Expect(volumes[2]).To(Equal("1")) }) diff --git a/test/e2e/systemd_test.go b/test/e2e/systemd_test.go index fb9e77b3da..143b8f59f0 100644 --- a/test/e2e/systemd_test.go +++ b/test/e2e/systemd_test.go @@ -81,13 +81,8 @@ WantedBy=multi-user.target }) It("podman run container with systemd PID1", func() { - systemdImage := "fedora" - pull := podmanTest.Podman([]string{"pull", systemdImage}) - pull.WaitWithDefaultTimeout() - Expect(pull.ExitCode()).To(Equal(0)) - ctrName := "testSystemd" - run := podmanTest.Podman([]string{"run", "--name", ctrName, "-t", "-i", "-d", systemdImage, "/usr/sbin/init"}) + run := podmanTest.Podman([]string{"run", "--name", ctrName, "-t", "-i", "-d", ubi_init, "/sbin/init"}) run.WaitWithDefaultTimeout() Expect(run.ExitCode()).To(Equal(0)) ctrID := run.OutputToString() From 7f8643a7820cb5bf3e8e196e31b1fa558b3a2a18 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Tue, 14 Jul 2020 11:01:12 -0500 Subject: [PATCH 3/4] [CI:DOCS]Do not copy policy.json into gating image test/policy.json should not need to be copied into the gating image Signed-off-by: Brent Baude --- contrib/gate/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/gate/Dockerfile b/contrib/gate/Dockerfile index 657b5accf4..9568038bb9 100644 --- a/contrib/gate/Dockerfile +++ b/contrib/gate/Dockerfile @@ -27,7 +27,6 @@ RUN set -x && \ # Install cni config COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist # Make sure we have some policy for pulling images -COPY test/policy.json /etc/containers/policy.json COPY test/redhat_sigstore.yaml /etc/containers/registries.d/registry.access.redhat.com.yaml WORKDIR "$GOSRC" From 660208cea38fdebb302f674b40bfd755f02def2d Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Tue, 16 Jun 2020 14:06:55 +0200 Subject: [PATCH 4/4] search: allow wildcards Allow wildcards in the search term. Note that not all registries support wildcards and it may only work with v1 registries. Note that searching implies figuring out if the specified search term includes a registry. If there's not registry detected, the search term will be used against all configured "unqualified-serach-registries" in the registries.conf. The parsing logic considers a registry to be the substring before the first slash `/`. With these changes we now not only support wildcards but arbitrary input; ultimately it's up to the registries to decide whether they support given input or not. Fixes: bugzilla.redhat.com/show_bug.cgi?id=1846629 Cherry-pick-of: commit b05888a97dbb Signed-off-by: Valentin Rothberg --- libpod/image/search.go | 17 ++++++++++------- pkg/registries/registries.go | 16 ---------------- test/e2e/search_test.go | 12 ++++++++++++ 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/libpod/image/search.go b/libpod/image/search.go index 0950921ed7..ee1ff0312b 100644 --- a/libpod/image/search.go +++ b/libpod/image/search.go @@ -64,13 +64,16 @@ type SearchFilter struct { // SearchImages searches images based on term and the specified SearchOptions // in all registries. func SearchImages(term string, options SearchOptions) ([]SearchResult, error) { - // Check if search term has a registry in it - registry, err := sysreg.GetRegistry(term) - if err != nil { - return nil, errors.Wrapf(err, "error getting registry from %q", term) - } - if registry != "" { - term = term[len(registry)+1:] + registry := "" + + // Try to extract a registry from the specified search term. We + // consider everything before the first slash to be the registry. Note + // that we cannot use the reference parser from the containers/image + // library as the search term may container arbitrary input such as + // wildcards. See bugzilla.redhat.com/show_bug.cgi?id=1846629. + if spl := strings.SplitN(term, "/", 2); len(spl) > 1 { + registry = spl[0] + term = spl[1] } registries, err := getRegistries(registry) diff --git a/pkg/registries/registries.go b/pkg/registries/registries.go index 927e7a62a8..ab7accd5ed 100644 --- a/pkg/registries/registries.go +++ b/pkg/registries/registries.go @@ -3,12 +3,10 @@ package registries import ( "os" "path/filepath" - "strings" "github.com/containers/image/v5/pkg/sysregistriesv2" "github.com/containers/image/v5/types" "github.com/containers/libpod/v2/pkg/rootless" - "github.com/docker/distribution/reference" "github.com/pkg/errors" ) @@ -77,17 +75,3 @@ func GetInsecureRegistries() ([]string, error) { } return insecureRegistries, nil } - -// GetRegistry returns the registry name from a string if specified -func GetRegistry(image string) (string, error) { - // It is possible to only have the registry name in the format "myregistry/" - // if so, just trim the "/" from the end and return the registry name - if strings.HasSuffix(image, "/") { - return strings.TrimSuffix(image, "/"), nil - } - imgRef, err := reference.Parse(image) - if err != nil { - return "", err - } - return reference.Domain(imgRef.(reference.Named)), nil -} diff --git a/test/e2e/search_test.go b/test/e2e/search_test.go index 2848e52cea..c62e0fabd6 100644 --- a/test/e2e/search_test.go +++ b/test/e2e/search_test.go @@ -400,4 +400,16 @@ registries = ['{{.Host}}:{{.Port}}']` search.WaitWithDefaultTimeout() Expect(search.ExitCode()).To(Not(Equal(0))) }) + + It("podman search with wildcards", func() { + search := podmanTest.Podman([]string{"search", "--limit", "30", "registry.redhat.io/*"}) + search.WaitWithDefaultTimeout() + Expect(search.ExitCode()).To(Equal(0)) + Expect(len(search.OutputToStringArray())).To(Equal(31)) + + search = podmanTest.Podman([]string{"search", "registry.redhat.io/*openshift*"}) + search.WaitWithDefaultTimeout() + Expect(search.ExitCode()).To(Equal(0)) + Expect(len(search.OutputToStringArray()) > 1).To(BeTrue()) + }) })