Add EROFS CI

.github/workflows/ci-build-image.yml

@@ -14,7 +14,7 @@ on:
 
 env:
   GO_VERSION: "1.22.0"
-  OBD_VERSION: "1.0.9"
+  OBD_VERSION: "1.0.12"
 
 jobs:
   build-image:

ci/build_image/Dockerfile

@@ -53,13 +53,13 @@ RUN apt-get update && apt-get install -y apt-transport-https ca-certificates cur
     apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io && \
     apt-get install -y libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
-    wget https://github.com/containerd/overlaybd/releases/download/v${OBD_VERSION}/overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \
-    dpkg -i overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \
+    wget https://github.com/containerd/overlaybd/releases/download/v${OBD_VERSION}/overlaybd-${OBD_VERSION}-20240717.b5b704b.ubuntu1.22.04.x86_64.deb && \
+    dpkg -i overlaybd-${OBD_VERSION}-20240717.b5b704b.ubuntu1.22.04.x86_64.deb && \
     dpkg -i overlaybd-snapshotter_${RELEASE_VERSION}_amd64.deb && \
     sed -i 's/"autoRemoveDev": false,/"autoRemoveDev": true,/g' /etc/overlaybd-snapshotter/config.json && \
     cat /etc/overlaybd-snapshotter/config.json && \
     mkdir -p /etc/containerd/ && \
     echo -e '[proxy_plugins.overlaybd]\n\ttype = "snapshot"\n\taddress = "/run/overlaybd-snapshotter/overlaybd.sock"' | tee -a /etc/containerd/config.toml && \
     cat /etc/containerd/config.toml && \
     chmod +x /app/start_services.sh && \
-    cat /app/start_services.sh
+    cat /app/start_services.sh

ci/uconv_reproduce/centos/config-turbo-erofs.json

@@ -0,0 +1 @@
+{"created":"2021-09-15T18:20:23.99863383Z","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/bash"],"Labels":{"org.label-schema.build-date":"20201113","org.label-schema.license":"GPLv2","org.label-schema.name":"CentOS Base Image","org.label-schema.schema-version":"1.0","org.label-schema.vendor":"CentOS","org.opencontainers.image.created":"2020-11-13 00:00:00+00:00","org.opencontainers.image.licenses":"GPL-2.0-only","org.opencontainers.image.title":"CentOS Base Image","org.opencontainers.image.vendor":"CentOS"}},"rootfs":{"type":"layers","diff_ids":["sha256:379d33b3da38f8546c995fe8977c6453c6d87e6671426f516b9a6357fa386ea8"]},"history":[{"created":"2021-09-15T18:20:23.417639551Z","created_by":"/bin/sh -c #(nop) ADD file:b3ebbe8bd304723d43b7b44a6d990cd657b63d93d6a2a9293983a30bfc1dfa53 in / "},{"created":"2021-09-15T18:20:23.819893035Z","created_by":"/bin/sh -c #(nop)  LABEL org.label-schema.schema-version=1.0 org.label-schema.name=CentOS Base Image org.label-schema.vendor=CentOS org.label-schema.license=GPLv2 org.label-schema.build-date=20201113 org.opencontainers.image.title=CentOS Base Image org.opencontainers.image.vendor=CentOS org.opencontainers.image.licenses=GPL-2.0-only org.opencontainers.image.created=2020-11-13 00:00:00+00:00","empty_layer":true},{"created":"2021-09-15T18:20:23.99863383Z","created_by":"/bin/sh -c #(nop)  CMD [\"/bin/bash\"]","empty_layer":true}]}

ci/uconv_reproduce/centos/manifest-turbo-erofs.json

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:48b560cf624c64f4864f769a3607f139a79fb592f33e1d3594f3e3c6d9259e26","size":1476},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:f64a7fa8973910cd8b6dd6884be20c20716baece12fedf90d1def44303040f4a","size":2697067,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:f64a7fa8973910cd8b6dd6884be20c20716baece12fedf90d1def44303040f4a","containerd.io/snapshot/overlaybd/blob-size":"2697067","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:2d473b07cdd5f0912cd6f1a703352c82b512407db6b05b43f2553732b55df3bc","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}}]}

ci/uconv_reproduce/ci-uconv-reproduce.sh

@@ -22,12 +22,16 @@ do
 
     tag_obd="${tag}_overlaybd"
     tag_turbo="${tag}_turbo"
+    tag_turbo_erofs="${tag}_turbo_erofs"
     manifest_obd="${workspace}/manifest.json"
     manifest_turbo="${workspace}/manifest-turbo.json"
+    manifest_turbo_erofs="${workspace}/manifest-turbo-erofs.json"
     config_obd="${workspace}/config.json"
     config_turbo="${workspace}/config-turbo.json"
+    config_turbo_erofs="${workspace}/config-turbo-erofs.json"
     output_obd="${workspace}/convert.overlaybd.out"
     output_turbo="${workspace}/convert.turbo.out"
+    output_turbo_erofs="${workspace}/convert.turbo.erofs.out"
 
     ${convertor} -r "${registry}/${img}" -i "${tag}" --overlaybd "${tag_obd}" -d "${workspace}/overlaybd_tmp_conv" &> "${output_obd}"
     curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_obd}" "https://${registry}/v2/${img}/manifests/${tag_obd}" &> /dev/null
@@ -41,11 +45,18 @@ do
     configDigest=${configDigest//\"/}
     curl -o "${config_turbo}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null
 
+    ${convertor} -r "${registry}/${img}" -i "${tag}" --turboOCI "${tag_turbo_erofs}" --fstype erofs -d "${workspace}/turbo_erofs_tmp_conv" &> "${output_turbo_erofs}"
+    curl -s -H "Accept: application/vnd.docker.distribution.manifest.v2+json" "https://${registry}/v2/${img}/manifests/${tag_turbo_erofs}"
+    curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_turbo_erofs}" "https://${registry}/v2/${img}/manifests/${tag_turbo_erofs}" &> /dev/null
+    configDigest=$(jq '.config.digest' "${manifest_turbo_erofs}")
+    configDigest=${configDigest//\"/}
+    curl -o "${config_turbo_erofs}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null
+
     prefix=$(date +%Y%m%d%H%M%S)
 
-    mode=("manifest" "config" "manifest" "config")
-    actual=("${manifest_obd}" "${config_obd}" "${manifest_turbo}" "${config_turbo}")
-    expected=("${ci_base}/${img}/manifest.json" "${ci_base}/${img}/config.json" "${ci_base}/${img}/manifest-turbo.json" "${ci_base}/${img}/config-turbo.json")
+    mode=("manifest" "config" "manifest" "config" "manifest" "config")
+    actual=("${manifest_obd}" "${config_obd}" "${manifest_turbo}" "${config_turbo}" "${manifest_turbo_erofs}" "${config_turbo_erofs}")
+    expected=("${ci_base}/${img}/manifest.json" "${ci_base}/${img}/config.json" "${ci_base}/${img}/manifest-turbo.json" "${ci_base}/${img}/config-turbo.json" "${ci_base}/${img}/manifest-turbo-erofs.json" "${ci_base}/${img}/config-turbo-erofs.json")
 
     conv_res=0
     n=${#mode[@]}
@@ -67,4 +78,4 @@ do
     fi
 done
 
-exit ${result}
+exit ${result}

ci/uconv_reproduce/nginx/config-turbo-erofs.json

@@ -0,0 +1 @@
+{"created":"2023-10-24T22:44:45Z","architecture":"amd64","os":"linux","config":{"ExposedPorts":{"80/tcp":{}},"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","NGINX_VERSION=1.25.3","NJS_VERSION=0.8.2","PKG_RELEASE=1~bookworm"],"Entrypoint":["/docker-entrypoint.sh"],"Cmd":["nginx","-g","daemon off;"],"Labels":{"maintainer":"NGINX Docker Maintainers \[email protected]\u003e"},"StopSignal":"SIGQUIT","ArgsEscaped":true},"rootfs":{"type":"layers","diff_ids":["sha256:abb8474cd5ee7c0199099cb2f09b7b5039cae70c74042f6809059bbaca34d94a","sha256:2b33f144d6b15dd87b62b370fa81ffb3eed54af9dd37ebe8f477d8852675957e","sha256:586f5f1ae7cc13a219018cc8f20070bc233eb8cb7d153b0b249167c26332a127","sha256:461b9e691ecf8a12b9578d480a5adb3e4c2f2d14fc39b684113f001e635a4bcf","sha256:1004fcd5fd6ff6abff91febdb5302caa3beef80c74c233060f50407a8a1ef8f3","sha256:10cbe3c0fa7fc18da12dc5a6d4a2499e19781bc3c46d62f6ceba1ac0eb3bd462","sha256:2bafba57f3e89d4e47b97141db54b20862e4c57e241732cd31dcdc3245c9998e"]},"history":[{"created":"2023-10-24T22:44:45Z","created_by":"/bin/sh -c #(nop) ADD file:ac3cd70031d35e46d86b876934946ffc8756de4de065fbc926dce642dac07ff3 in / "},{"created":"2023-10-24T22:44:45Z","created_by":"/bin/sh -c #(nop)  CMD [\"bash\"]","empty_layer":true},{"created":"2023-10-24T22:44:45Z","created_by":"LABEL maintainer=NGINX Docker Maintainers \[email protected]\u003e","comment":"buildkit.dockerfile.v0","empty_layer":true},{"created":"2023-10-24T22:44:45Z","created_by":"ENV NGINX_VERSION=1.25.3","comment":"buildkit.dockerfile.v0","empty_layer":true},{"created":"2023-10-24T22:44:45Z","created_by":"ENV NJS_VERSION=0.8.2","comment":"buildkit.dockerfile.v0","empty_layer":true},{"created":"2023-10-24T22:44:45Z","created_by":"ENV PKG_RELEASE=1~bookworm","comment":"buildkit.dockerfile.v0","empty_layer":true},{"created":"2023-10-24T22:44:45Z","created_by":"RUN /bin/sh -c set -x     \u0026\u0026 groupadd --system --gid 101 nginx     \u0026\u0026 useradd --system --gid nginx --no-create-home --home /nonexistent --comment \"nginx user\" --shell /bin/false --uid 101 nginx     \u0026\u0026 apt-get update     \u0026\u0026 apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates     \u0026\u0026     NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62;     NGINX_GPGKEY_PATH=/usr/share/keyrings/nginx-archive-keyring.gpg;     export GNUPGHOME=\"$(mktemp -d)\";     found='';     for server in         hkp://keyserver.ubuntu.com:80         pgp.mit.edu     ; do         echo \"Fetching GPG key $NGINX_GPGKEY from $server\";         gpg1 --keyserver \"$server\" --keyserver-options timeout=10 --recv-keys \"$NGINX_GPGKEY\" \u0026\u0026 found=yes \u0026\u0026 break;     done;     test -z \"$found\" \u0026\u0026 echo \u003e\u00262 \"error: failed to fetch GPG key $NGINX_GPGKEY\" \u0026\u0026 exit 1;     gpg1 --export \"$NGINX_GPGKEY\" \u003e \"$NGINX_GPGKEY_PATH\" ;     rm -rf \"$GNUPGHOME\";     apt-get remove --purge --auto-remove -y gnupg1 \u0026\u0026 rm -rf /var/lib/apt/lists/*     \u0026\u0026 dpkgArch=\"$(dpkg --print-architecture)\"     \u0026\u0026 nginxPackages=\"         nginx=${NGINX_VERSION}-${PKG_RELEASE}         nginx-module-xslt=${NGINX_VERSION}-${PKG_RELEASE}         nginx-module-geoip=${NGINX_VERSION}-${PKG_RELEASE}         nginx-module-image-filter=${NGINX_VERSION}-${PKG_RELEASE}         nginx-module-njs=${NGINX_VERSION}+${NJS_VERSION}-${PKG_RELEASE}     \"     \u0026\u0026 case \"$dpkgArch\" in         amd64|arm64)             echo \"deb [signed-by=$NGINX_GPGKEY_PATH] https://nginx.org/packages/mainline/debian/ bookworm nginx\" \u003e\u003e /etc/apt/sources.list.d/nginx.list             \u0026\u0026 apt-get update             ;;         *)             echo \"deb-src [signed-by=$NGINX_GPGKEY_PATH] https://nginx.org/packages/mainline/debian/ bookworm nginx\" \u003e\u003e /etc/apt/sources.list.d/nginx.list                         \u0026\u0026 tempDir=\"$(mktemp -d)\"             \u0026\u0026 chmod 777 \"$tempDir\"                         \u0026\u0026 savedAptMark=\"$(apt-mark showmanual)\"                         \u0026\u0026 apt-get update             \u0026\u0026 apt-get build-dep -y $nginxPackages             \u0026\u0026 (                 cd \"$tempDir\"                 \u0026\u0026 DEB_BUILD_OPTIONS=\"nocheck parallel=$(nproc)\"                     apt-get source --compile $nginxPackages             )                         \u0026\u0026 apt-mark showmanual | xargs apt-mark auto \u003e /dev/null             \u0026\u0026 { [ -z \"$savedAptMark\" ] || apt-mark manual $savedAptMark; }                         \u0026\u0026 ls -lAFh \"$tempDir\"             \u0026\u0026 ( cd \"$tempDir\" \u0026\u0026 dpkg-scanpackages . \u003e Packages )             \u0026\u0026 grep '^Package: ' \"$tempDir/Packages\"             \u0026\u0026 echo \"deb [ trusted=yes ] file://$tempDir ./\" \u003e /etc/apt/sources.list.d/temp.list             \u0026\u0026 apt-get -o Acquire::GzipIndexes=false update             ;;     esac         \u0026\u0026 apt-get install --no-install-recommends --no-install-suggests -y                         $nginxPackages                         gettext-base                         curl     \u0026\u0026 apt-get remove --purge --auto-remove -y \u0026\u0026 rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list         \u0026\u0026 if [ -n \"$tempDir\" ]; then         apt-get purge -y --auto-remove         \u0026\u0026 rm -rf \"$tempDir\" /etc/apt/sources.list.d/temp.list;     fi     \u0026\u0026 ln -sf /dev/stdout /var/log/nginx/access.log     \u0026\u0026 ln -sf /dev/stderr /var/log/nginx/error.log     \u0026\u0026 mkdir /docker-entrypoint.d # buildkit","comment":"buildkit.dockerfile.v0"},{"created":"2023-10-24T22:44:45Z","created_by":"COPY docker-entrypoint.sh / # buildkit","comment":"buildkit.dockerfile.v0"},{"created":"2023-10-24T22:44:45Z","created_by":"COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d # buildkit","comment":"buildkit.dockerfile.v0"},{"created":"2023-10-24T22:44:45Z","created_by":"COPY 15-local-resolvers.envsh /docker-entrypoint.d # buildkit","comment":"buildkit.dockerfile.v0"},{"created":"2023-10-24T22:44:45Z","created_by":"COPY 20-envsubst-on-templates.sh /docker-entrypoint.d # buildkit","comment":"buildkit.dockerfile.v0"},{"created":"2023-10-24T22:44:45Z","created_by":"COPY 30-tune-worker-processes.sh /docker-entrypoint.d # buildkit","comment":"buildkit.dockerfile.v0"},{"created":"2023-10-24T22:44:45Z","created_by":"ENTRYPOINT [\"/docker-entrypoint.sh\"]","comment":"buildkit.dockerfile.v0","empty_layer":true},{"created":"2023-10-24T22:44:45Z","created_by":"EXPOSE map[80/tcp:{}]","comment":"buildkit.dockerfile.v0","empty_layer":true},{"created":"2023-10-24T22:44:45Z","created_by":"STOPSIGNAL SIGQUIT","comment":"buildkit.dockerfile.v0","empty_layer":true},{"created":"2023-10-24T22:44:45Z","created_by":"CMD [\"nginx\" \"-g\" \"daemon off;\"]","comment":"buildkit.dockerfile.v0","empty_layer":true}]}

ci/uconv_reproduce/nginx/manifest-turbo-erofs.json

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:67af1543970e694908655ff9bee5c6306c2bfebbd5b5e5f85e6df11a2a7c4bf9","size":7001},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:8669c0b06bac9a3b6437647361cc0fed70a500744e513bdbb4088c71db0c4c01","size":1104210,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:8669c0b06bac9a3b6437647361cc0fed70a500744e513bdbb4088c71db0c4c01","containerd.io/snapshot/overlaybd/blob-size":"1104210","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:af107e978371b6cd6339127a05502c5eacd1e6b0e9eb7b2f4aa7b6fc87e2dd81","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}},{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:41e8e4b81e936a1587c15ffadc01505fd8b53602d96d28973356753c905de8a9","size":1445588,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:41e8e4b81e936a1587c15ffadc01505fd8b53602d96d28973356753c905de8a9","containerd.io/snapshot/overlaybd/blob-size":"1445588","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:336ba1f05c3ede29f0a73d3f88b39a14f6abdc57fafedf3891fd793504440263","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}},{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:c254cf608f0631dca80975f3fc34de475578bc77191c719dbf26492920637915","size":972,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:c254cf608f0631dca80975f3fc34de475578bc77191c719dbf26492920637915","containerd.io/snapshot/overlaybd/blob-size":"972","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:8c37d2ff6efa0a08f83056109a47aa0caf2cc82136d926d1176cd451f7fbb245","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}},{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:e42484694a78760fc87cad61e5aebd8314c9d17cbdde7e6bfe68dfb72b03c438","size":1055,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:e42484694a78760fc87cad61e5aebd8314c9d17cbdde7e6bfe68dfb72b03c438","containerd.io/snapshot/overlaybd/blob-size":"1055","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:51d6357098de68f5fc2e50afdaa73fc4fcbdeed2161adc9f14d1d8dae9d94d36","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}},{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:f272d06d4ad1db1612834947ffa53c8afc10ffcbf2f26985ce971fe219fb94e8","size":1077,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:f272d06d4ad1db1612834947ffa53c8afc10ffcbf2f26985ce971fe219fb94e8","containerd.io/snapshot/overlaybd/blob-size":"1077","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:782f1ecce57d1fa61421872a16b979ad92057db19841b5811616a749705214f4","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}},{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:4d8463f6791599f5ecc6b75a9c0c663b397f2569c6c4e8090d741f8e6e5b8ec4","size":1117,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:4d8463f6791599f5ecc6b75a9c0c663b397f2569c6c4e8090d741f8e6e5b8ec4","containerd.io/snapshot/overlaybd/blob-size":"1117","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:5e99d351b073fec15b9817dc5234f32433ef0404849cc66857be2eca5192ccf8","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}},{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:1c0a24247098f7e92210357dabc96adad60fb88c79ee6bcdad8d50b4a2e632b7","size":1137,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:1c0a24247098f7e92210357dabc96adad60fb88c79ee6bcdad8d50b4a2e632b7","containerd.io/snapshot/overlaybd/blob-size":"1137","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:7b73345df136081ef2e60fd5cb875771c02c5ecb76015292babbc4711d195a31","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}}]}