test composefs on rawhide

.cirrus.yml

@@ -60,7 +60,7 @@ env:
     CTR_FQIN:                # One of the "Container FQIN's" (above)
     CI_DESIRED_RUNTIME: crun    # As of 2024-05-28 there are no other supported runtimes
     CI_DESIRED_DATABASE: sqlite # 'sqlite' or 'boltdb'
-    CI_DESIRED_STORAGE: overlay # overlay or vfs
+    CI_DESIRED_STORAGE: overlay # overlay, vfs, or composefs (which is actually overlay)
 
     # Curl-command prefix for downloading task artifacts, simply add the
     # the url-encoded task name, artifact name, and path as a suffix.
@@ -155,6 +155,7 @@ build_task:
               <<: *stdenvars
               DISTRO_NV: ${RAWHIDE_NAME}
               VM_IMAGE_NAME: ${RAWHIDE_CACHE_IMAGE_NAME}
+              CI_DESIRED_STORAGE: composefs
               CTR_FQIN: ""
         - env:
               DISTRO_NV: ${DEBIAN_NAME}

contrib/cirrus/lib.sh

@@ -88,7 +88,7 @@ PASSTHROUGH_ENV_EXACT='CGROUP_MANAGER|DEST_BRANCH|DISTRO_NV|GOCACHE|GOPATH|GOSRC
 
 # List of envariable patterns which must match AT THE BEGINNING of the name.
 # Consumed by the passthrough_envars() automation library function.
-PASSTHROUGH_ENV_ATSTART='CI|LANG|LC_|TEST'
+PASSTHROUGH_ENV_ATSTART='CI|LANG|LC_|STORAGE_OPTIONS_|TEST'
 
 # List of envariable patterns which can match ANYWHERE in the name.
 # Consumed by the passthrough_envars() automation library function.

contrib/cirrus/logformatter

@@ -591,10 +591,12 @@ END_HTML
             # Highlight the important (non-boilerplate) podman command.
             $line =~ s/\s+--remote\s+/ /g;      # --remote takes no args
             # Strip out the global podman options, but show them on hover
-            $line =~ s{(\S+\/podman(-remote)?)((\s+--(root|runroot|runtime|tmpdir|storage-opt|conmon|cgroup-manager|\S+-backend|network-config-dir|storage-driver|url) \S+)*)(\s.*)}{
+            $line =~ s{(\S+\/podman(-remote)?)((\s+--(root|runroot|runtime|tmpdir|storage-opt|conmon|cgroup-manager|\S+-backend|network-config-dir|pull-option|storage-driver|url) \S+)*)(\s.*)}{
                 my ($full_path, $remote, $options, $args) = ($1, $2||'', $3, $6);
 
                 $options =~ s/^\s+//;
+                # July 2024: "" in composefs pull-option confuses hovertext
+                $options =~ s/\"//g;
                 # Separate each '--foo bar' with newlines for readability
                 $options =~ s/ --/\n--/g;
                 qq{<span title="$full_path"><b>podman$remote</b></span> <span class=\"boring\" title=\"$options\">[options]</span><b>$args</b>};

contrib/cirrus/setup_environment.sh

@@ -41,6 +41,27 @@ done
 # Bypass git safety/security checks when operating in a throwaway environment
 showrun git config --global --add safe.directory $GOSRC
 
+# Special case: "composefs" is not a valid setting but it's useful for
+# readability in .cirrus.yml. Here we translate that to overlayfs (the
+# actual filesystem) along with extra magic envariables.
+# Be sure to do this before writing /etc/ci_environment.
+export CI_DESIRED_COMPOSEFS=
+# shellcheck disable=SC2154
+if [[ "$CI_DESIRED_STORAGE" = "composefs" ]]; then
+    CI_DESIRED_STORAGE="overlay"
+
+    # composefs is root only
+    if [[ "$PRIV_NAME" == "root" ]]; then
+        CI_DESIRED_COMPOSEFS="+composefs"
+
+        # KLUDGE ALERT! Magic options needed for testing composefs.
+        # This option was intended for passing one arg to --storage-opt
+        # but we're hijacking it to pass an extra option+arg. And it
+        # actually works. Just MAKE SURE THERE ARE NO SPACES IN THE {...}!
+        export STORAGE_OPTIONS_OVERLAY='overlay.use_composefs=true --pull-option {enable_partial_images="true",use_hard_links="false",ostree_repos="",convert_images="true"}'
+    fi
+fi
+
 # Ensure that all lower-level contexts and child-processes have
 # ready access to higher level orchestration (e.g Cirrus-CI)
 # variables.
@@ -155,7 +176,7 @@ esac
 # This is (sigh) different because e2e tests have their own special way
 # of ignoring system defaults.
 # shellcheck disable=SC2154
-showrun echo "Setting CI_DESIRED_STORAGE [=$CI_DESIRED_STORAGE] for *system* tests"
+showrun echo "Setting CI_DESIRED_STORAGE [=$CI_DESIRED_STORAGE$CI_DESIRED_COMPOSEFS] for *system* tests"
 conf=/etc/containers/storage.conf
 if [[ -e $conf ]]; then
     die "FATAL! INTERNAL ERROR! Cannot override $conf"
@@ -167,6 +188,18 @@ runroot = "/run/containers/storage"
 graphroot = "/var/lib/containers/storage"
 EOF
 
+if [[ -n "$CI_DESIRED_COMPOSEFS" ]]; then
+    cat <<EOF >>$conf
+
+# BEGIN CI-enabled composefs
+[storage.options]
+pull_options = {enable_partial_images = "true", use_hard_links = "false", ostree_repos="", convert_images = "true"}
+
+[storage.options.overlay]
+use_composefs = "true"
+# END CI-enabled composefs
+EOF
+fi
 
 # mount a tmpfs for the container storage to speed up the IO
 # side effect is we clear all potentially pre existing data so we know we always start "clean"

docs/source/markdown/podman.1.md

@@ -263,7 +263,7 @@ Set default `--storage-driver` value.
 
 #### **STORAGE_OPTS**
 
-Set default `--storage-opts` value.
+Set default `--storage-opt` value.
 
 #### **TMPDIR**
 

test/e2e/common_test.go

@@ -266,11 +266,6 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 		}
 	}
 
-	storageOptions := os.Getenv("STORAGE_OPTIONS")
-	if storageOptions == "" {
-		storageOptions = STORAGE_OPTIONS
-	}
-
 	cgroupManager := os.Getenv("CGROUP_MANAGER")
 	if cgroupManager == "" {
 		cgroupManager = CGROUP_MANAGER
@@ -313,9 +308,17 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 	if isRootless() {
 		storageFs = ROOTLESS_STORAGE_FS
 	}
+
+	storageOptions := STORAGE_OPTIONS
 	if os.Getenv("STORAGE_FS") != "" {
 		storageFs = os.Getenv("STORAGE_FS")
 		storageOptions = "--storage-driver " + storageFs
+
+		// Look for STORAGE_OPTIONS_OVERLAY / STORAGE_OPTIONS_VFS
+		extraOptions := os.Getenv("STORAGE_OPTIONS_" + strings.ToUpper(storageFs))
+		if extraOptions != "" {
+			storageOptions += " --storage-opt " + extraOptions
+		}
 	}
 
 	p := &PodmanTestIntegration{

test/e2e/config_amd64.go

@@ -1,15 +1,14 @@
 package integration
 
 var (
-	STORAGE_FS               = "overlay"                                                                                                                   //nolint:revive,stylecheck
-	STORAGE_OPTIONS          = "--storage-driver overlay"                                                                                                  //nolint:revive,stylecheck
-	ROOTLESS_STORAGE_FS      = "overlay"                                                                                                                   //nolint:revive,stylecheck
-	ROOTLESS_STORAGE_OPTIONS = "--storage-driver overlay"                                                                                                  //nolint:revive,stylecheck
-	CACHE_IMAGES             = []string{ALPINE, BB, NGINX_IMAGE, REDIS_IMAGE, REGISTRY_IMAGE, INFRA_IMAGE, CITEST_IMAGE, HEALTHCHECK_IMAGE, SYSTEMD_IMAGE} //nolint:revive,stylecheck
-	NGINX_IMAGE              = "quay.io/libpod/alpine_nginx:latest"                                                                                        //nolint:revive,stylecheck
-	BB_GLIBC                 = "docker.io/library/busybox:glibc"                                                                                           //nolint:revive,stylecheck
-	REGISTRY_IMAGE           = "quay.io/libpod/registry:2.8.2"                                                                                             //nolint:revive,stylecheck
-	CITEST_IMAGE             = "quay.io/libpod/testimage:20240123"                                                                                         //nolint:revive,stylecheck
-	SYSTEMD_IMAGE            = "quay.io/libpod/systemd-image:20240124"                                                                                     //nolint:revive,stylecheck
-	CIRROS_IMAGE             = "quay.io/libpod/cirros:latest"                                                                                              //nolint:revive,stylecheck
+	STORAGE_FS          = "overlay"                                                                                                                   //nolint:revive,stylecheck
+	STORAGE_OPTIONS     = "--storage-driver overlay"                                                                                                  //nolint:revive,stylecheck
+	ROOTLESS_STORAGE_FS = "overlay"                                                                                                                   //nolint:revive,stylecheck
+	CACHE_IMAGES        = []string{ALPINE, BB, NGINX_IMAGE, REDIS_IMAGE, REGISTRY_IMAGE, INFRA_IMAGE, CITEST_IMAGE, HEALTHCHECK_IMAGE, SYSTEMD_IMAGE} //nolint:revive,stylecheck
+	NGINX_IMAGE         = "quay.io/libpod/alpine_nginx:latest"                                                                                        //nolint:revive,stylecheck
+	BB_GLIBC            = "docker.io/library/busybox:glibc"                                                                                           //nolint:revive,stylecheck
+	REGISTRY_IMAGE      = "quay.io/libpod/registry:2.8.2"                                                                                             //nolint:revive,stylecheck
+	CITEST_IMAGE        = "quay.io/libpod/testimage:20240123"                                                                                         //nolint:revive,stylecheck
+	SYSTEMD_IMAGE       = "quay.io/libpod/systemd-image:20240124"                                                                                     //nolint:revive,stylecheck
+	CIRROS_IMAGE        = "quay.io/libpod/cirros:latest"                                                                                              //nolint:revive,stylecheck
 )

test/e2e/config_arm64.go

@@ -1,15 +1,14 @@
 package integration
 
 var (
-	STORAGE_FS               = "overlay"                                                                                                                                  //nolint:revive,stylecheck
-	STORAGE_OPTIONS          = "--storage-driver overlay"                                                                                                                 //nolint:revive,stylecheck
-	ROOTLESS_STORAGE_FS      = "overlay"                                                                                                                                  //nolint:revive,stylecheck
-	ROOTLESS_STORAGE_OPTIONS = "--storage-driver overlay"                                                                                                                 //nolint:revive,stylecheck
-	CACHE_IMAGES             = []string{ALPINE, BB, fedoraMinimal, NGINX_IMAGE, REDIS_IMAGE, REGISTRY_IMAGE, INFRA_IMAGE, CITEST_IMAGE, HEALTHCHECK_IMAGE, SYSTEMD_IMAGE} //nolint:revive,stylecheck
-	NGINX_IMAGE              = "quay.io/lsm5/alpine_nginx-aarch64:latest"                                                                                                 //nolint:revive,stylecheck
-	BB_GLIBC                 = "docker.io/library/busybox:glibc"                                                                                                          //nolint:revive,stylecheck
-	REGISTRY_IMAGE           = "quay.io/libpod/registry:2.8.2"                                                                                                            //nolint:revive,stylecheck
-	CITEST_IMAGE             = "quay.io/libpod/testimage:20240123"                                                                                                        //nolint:revive,stylecheck
-	SYSTEMD_IMAGE            = "quay.io/libpod/systemd-image:20240124"                                                                                                    //nolint:revive,stylecheck
-	CIRROS_IMAGE             = "quay.io/libpod/cirros:latest"                                                                                                             //nolint:revive,stylecheck
+	STORAGE_FS          = "overlay"                                                                                                                                  //nolint:revive,stylecheck
+	STORAGE_OPTIONS     = "--storage-driver overlay"                                                                                                                 //nolint:revive,stylecheck
+	ROOTLESS_STORAGE_FS = "overlay"                                                                                                                                  //nolint:revive,stylecheck
+	CACHE_IMAGES        = []string{ALPINE, BB, fedoraMinimal, NGINX_IMAGE, REDIS_IMAGE, REGISTRY_IMAGE, INFRA_IMAGE, CITEST_IMAGE, HEALTHCHECK_IMAGE, SYSTEMD_IMAGE} //nolint:revive,stylecheck
+	NGINX_IMAGE         = "quay.io/lsm5/alpine_nginx-aarch64:latest"                                                                                                 //nolint:revive,stylecheck
+	BB_GLIBC            = "docker.io/library/busybox:glibc"                                                                                                          //nolint:revive,stylecheck
+	REGISTRY_IMAGE      = "quay.io/libpod/registry:2.8.2"                                                                                                            //nolint:revive,stylecheck
+	CITEST_IMAGE        = "quay.io/libpod/testimage:20240123"                                                                                                        //nolint:revive,stylecheck
+	SYSTEMD_IMAGE       = "quay.io/libpod/systemd-image:20240124"                                                                                                    //nolint:revive,stylecheck
+	CIRROS_IMAGE        = "quay.io/libpod/cirros:latest"                                                                                                             //nolint:revive,stylecheck
 )

test/e2e/config_ppc64le.go

@@ -1,13 +1,12 @@
 package integration
 
 var (
-	STORAGE_FS               = "overlay"
-	STORAGE_OPTIONS          = "--storage-driver overlay"
-	ROOTLESS_STORAGE_FS      = "overlay"
-	ROOTLESS_STORAGE_OPTIONS = "--storage-driver overlay"
-	CACHE_IMAGES             = []string{ALPINE, BB, fedoraMinimal, NGINX_IMAGE, REDIS_IMAGE, INFRA_IMAGE, CITEST_IMAGE}
-	NGINX_IMAGE              = "quay.io/libpod/alpine_nginx-ppc64le:latest"
-	BB_GLIBC                 = "docker.io/ppc64le/busybox:glibc"
-	CITEST_IMAGE             = "quay.io/libpod/testimage:20240123"
-	REGISTRY_IMAGE           string
+	STORAGE_FS          = "overlay"
+	STORAGE_OPTIONS     = "--storage-driver overlay"
+	ROOTLESS_STORAGE_FS = "overlay"
+	CACHE_IMAGES        = []string{ALPINE, BB, fedoraMinimal, NGINX_IMAGE, REDIS_IMAGE, INFRA_IMAGE, CITEST_IMAGE}
+	NGINX_IMAGE         = "quay.io/libpod/alpine_nginx-ppc64le:latest"
+	BB_GLIBC            = "docker.io/ppc64le/busybox:glibc"
+	CITEST_IMAGE        = "quay.io/libpod/testimage:20240123"
+	REGISTRY_IMAGE      string
 )

test/e2e/info_test.go

@@ -240,6 +240,18 @@ var _ = Describe("Podman Info", func() {
 		session.WaitWithDefaultTimeout()
 		Expect(session).To(ExitCleanly())
 		Expect(session.OutputToString()).To(Equal(want), ".Store.GraphDriverName from podman info")
+
+		// Confirm desired setting of composefs
+		if want == "overlay" {
+			expect := "<no value>"
+			if os.Getenv("CI_DESIRED_COMPOSEFS") != "" {
+				expect = "true"
+			}
+			session = podmanTest.Podman([]string{"info", "--format", `{{index .Store.GraphOptions "overlay.use_composefs"}}`})
+			session.WaitWithDefaultTimeout()
+			Expect(session).To(ExitCleanly())
+			Expect(session.OutputToString()).To(Equal(expect), ".Store.GraphOptions -> overlay.use_composefs")
+		}
 	})
 
 	It("Podman info: check lock count", Serial, func() {

test/system/005-info.bats

@@ -124,6 +124,16 @@ host.slirp4netns.executable | $expr_path
     fi
 
     is "$(podman_storage_driver)" "$CI_DESIRED_STORAGE" "podman storage driver is not CI_DESIRED_STORAGE (from .cirrus.yml)"
+
+    # Confirm desired setting of composefs
+    if [[ "$CI_DESIRED_STORAGE" = "overlay" ]]; then
+        expect="<no value>"
+        if [[ -n "$CI_DESIRED_COMPOSEFS" ]]; then
+            expect="true"
+        fi
+        run_podman info --format '{{index .Store.GraphOptions "overlay.use_composefs"}}'
+        assert "$output" = "$expect" ".Store.GraphOptions -> overlay.use_composefs"
+    fi
 }
 
 # 2021-04-06 discussed in watercooler: RHEL must never use crun, even if

test/system/010-images.bats

@@ -360,6 +360,12 @@ driver="$storagedriver"
 additionalimagestores = [ "$imstore/root" ]
 EOF
 
+    # If composefs (root only) is enabled, we must mirror that setting in our conf
+    if grep -q 'BEGIN CI-enabled composefs' /etc/containers/storage.conf; then
+        sed -ne '/BEGIN CI-enabled composefs/,/END CI-enabled composefs/p' /etc/containers/storage.conf \
+            | grep -vF '[storage.options]' >>$sconf
+    fi
+
     skopeo copy containers-storage:$IMAGE \
            containers-storage:\[${storagedriver}@${imstore}/root+${imstore}/runroot\]$IMAGE