diff --git a/.github/workflows/ci-userspace-convertor.yml b/.github/workflows/ci-userspace-convertor.yml index 2a5350db..3c5a0f6d 100644 --- a/.github/workflows/ci-userspace-convertor.yml +++ b/.github/workflows/ci-userspace-convertor.yml @@ -26,7 +26,6 @@ jobs: - /dev:/dev - /lib/modules:/lib/modules - /sys/kernel/config:/sys/kernel/config - - /mnt:/var/lib/containerd options: --privileged steps: @@ -41,12 +40,29 @@ jobs: shell: bash run: | bash start_services.sh - sleep 3s - docker run -d -p 5000:5000 --restart always --name registry registry:2 - sleep 3s + sleep 5s + + - name: Prepare Local Registry + working-directory: ci/scripts + shell: bash + run: | + bash new_registry.sh + bash prepare_image.sh registry.hub.docker.com/overlaybd/centos:centos7.9.2009 localhost:5000/centos:centos7.9.2009 && \ + bash prepare_image.sh registry.hub.docker.com/overlaybd/ubuntu:22.04 localhost:5000/ubuntu:22.04 && \ + bash prepare_image.sh registry.hub.docker.com/overlaybd/redis:7.2.3 localhost:5000/redis:7.2.3 && \ + bash prepare_image.sh registry.hub.docker.com/overlaybd/wordpress:6.4.2 localhost:5000/wordpress:6.4.2 && \ + bash prepare_image.sh registry.hub.docker.com/overlaybd/nginx:1.25.3 localhost:5000/nginx:1.25.3 - name: CI - uconv reproduce working-directory: ci/uconv_reproduce shell: bash run: | bash ci-uconv-reproduce.sh + + - name: CI - uconv E2E + working-directory: ci/scripts + shell: bash + run: | + /opt/overlaybd/snapshotter/convertor -r localhost:5000/redis -i 7.2.3 --overlaybd 7.2.3_overlaybd --turboOCI 7.2.3_turbo + bash run_container.sh localhost:5000/redis:7.2.3_overlaybd + bash run_container.sh localhost:5000/redis:7.2.3_turbo diff --git a/ci/build_image/Dockerfile b/ci/build_image/Dockerfile index ce4d9b5f..f465b6ec 100644 --- a/ci/build_image/Dockerfile +++ b/ci/build_image/Dockerfile @@ -48,10 +48,8 @@ RUN apt-get update && apt-get install -y apt-transport-https ca-certificates cur "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -RUN \ - apt update && apt install -y libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \ + apt-get install -y libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ wget https://github.com/containerd/overlaybd/releases/download/v${OBD_VERSION}/overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \ dpkg -i overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \ dpkg -i overlaybd-snapshotter_${RELEASE_VERSION}_amd64.deb && \ diff --git a/ci/scripts/new_registry.sh b/ci/scripts/new_registry.sh new file mode 100755 index 00000000..aa932057 --- /dev/null +++ b/ci/scripts/new_registry.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# run a HTTPS registry + +set -x + +rm -rf /etc/registry/ +mkdir -p /etc/registry/certs/ +mkdir -p /etc/registry/config/ + +# generate server certifications +cat << EOF > /etc/registry/openssl.cnf +[req] +distinguished_name = req_distinguished_name +x509_extensions = v3_req +prompt = no + +[req_distinguished_name] +C = CN +ST = Beijing +L = Beijing City +O = Alibaba +CN = localhost + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = localhost +IP.1 = 127.0.0.1 +EOF + +openssl req -new -x509 -newkey rsa:2048 -sha256 -nodes -config /etc/registry/openssl.cnf \ + -days 365 -out /etc/registry/certs/server.crt -keyout /etc/registry/certs/server.key + +ls /etc/registry/certs/ +cp /etc/registry/certs/server.crt /usr/local/share/ca-certificates/registry.crt +update-ca-certificates + +# start registry +cat << EOF > /etc/registry/config/config.yml +version: 0.1 +log: + fields: + service: registry +storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] + tls: + certificate: /certs/server.crt + key: /certs/server.key +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +EOF + +docker run -d --restart=always --name registry -p 5000:5000 \ + -v /etc/registry/certs:/certs \ + -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/server.crt \ + -e REGISTRY_HTTP_TLS_KEY=/certs/server.key \ + registry:2 + +sleep 5s + +docker ps -a +apt-get update && apt-get install -y lsof +lsof -i :5000 +curl http://localhost:5000/v2/_catalog +lsof -i :5000 +curl https://localhost:5000/v2/_catalog diff --git a/ci/scripts/prepare_image.sh b/ci/scripts/prepare_image.sh new file mode 100755 index 00000000..9b2484ba --- /dev/null +++ b/ci/scripts/prepare_image.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +from=${1:?} +to=${2:?} + +set -x + +ctr i pull "${from}" +ctr i tag "${from}" "${to}" +ctr i push "${to}" +ctr i rm "${from}" "${to}" diff --git a/ci/scripts/run_container.sh b/ci/scripts/run_container.sh new file mode 100755 index 00000000..17886dae --- /dev/null +++ b/ci/scripts/run_container.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# +# rpull and run on-demand + +image=$1 +container_name=${2:-test} + +/opt/overlaybd/snapshotter/ctr rpull "${image}" +ctr run -d --net-host --snapshotter=overlaybd "${image}" "${container_name}" +ctr t ls | grep "${container_name}" +ctr t kill -s 9 "${container_name}" && sleep 5s && ctr t ls +ctr c rm "${container_name}" && ctr c ls +ctr i rm "${image}" diff --git a/ci/uconv_reproduce/ci-uconv-reproduce.sh b/ci/uconv_reproduce/ci-uconv-reproduce.sh index c12bd4cd..7a22d581 100644 --- a/ci/uconv_reproduce/ci-uconv-reproduce.sh +++ b/ci/uconv_reproduce/ci-uconv-reproduce.sh @@ -1,6 +1,6 @@ #!/bin/bash -apt install -y python3 jq +apt update && apt install -y python3 jq convertor="/opt/overlaybd/snapshotter/convertor" images=("centos:centos7.9.2009" "ubuntu:22.04" "redis:7.2.3" "wordpress:6.4.2" "nginx:1.25.3") @@ -11,11 +11,6 @@ result=0 for image in "${images[@]}" do - from_img="registry.hub.docker.com/overlaybd/${image}" - ctr i pull "${from_img}" &> /dev/null - ctr i tag "${from_img}" "${registry}/${image}" &> /dev/null - ctr i push "${registry}/${image}" &> /dev/null - img=${image%%":"*} tag=${image##*":"} echo "${img} ${tag}" @@ -35,16 +30,16 @@ do output_turbo="${workspace}/convert.turbo.out" ${convertor} -r "${registry}/${img}" -i "${tag}" --overlaybd "${tag_obd}" -d "${workspace}/overlaybd_tmp_conv" &> "${output_obd}" - curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_obd}" "${registry}/v2/${img}/manifests/${tag_obd}" &> /dev/null + curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_obd}" "https://${registry}/v2/${img}/manifests/${tag_obd}" configDigest=$(jq '.config.digest' "${manifest_obd}") configDigest=${configDigest//\"/} - curl -o "${config_obd}" "${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null + curl -o "${config_obd}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null ${convertor} -r "${registry}/${img}" -i "${tag}" --turboOCI "${tag_turbo}" -d "${workspace}/turbo_tmp_conv" &> "${output_turbo}" - curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_turbo}" "${registry}/v2/${img}/manifests/${tag_turbo}" &> /dev/null + curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_turbo}" "https://${registry}/v2/${img}/manifests/${tag_turbo}" &> /dev/null configDigest=$(jq '.config.digest' "${manifest_turbo}") configDigest=${configDigest//\"/} - curl -o "${config_turbo}" "${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null + curl -o "${config_turbo}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null prefix=$(date +%Y%m%d%H%M%S) diff --git a/ci/uconv_reproduce/compare_layers.py b/ci/uconv_reproduce/compare_layers.py index 29c04388..0e3d46d5 100644 --- a/ci/uconv_reproduce/compare_layers.py +++ b/ci/uconv_reproduce/compare_layers.py @@ -48,10 +48,10 @@ def main(): fb = sys.argv[3] if not os.path.exists(fa): print("file %s not exist" % fa) - return -1 + sys.exit(-1) if not os.path.exists(fb): print("file %s not exist" % fb) - return -1 + sys.exit(-1) fa_conf = json.load(open(fa, 'r')) fb_conf = json.load(open(fb, 'r')) if ftype == "manifest": diff --git a/cmd/convertor/builder/builder.go b/cmd/convertor/builder/builder.go index 7cb651ed..1ee514a0 100644 --- a/cmd/convertor/builder/builder.go +++ b/cmd/convertor/builder/builder.go @@ -83,17 +83,28 @@ func NewOverlayBDBuilder(ctx context.Context, opt BuilderOptions) (Builder, erro TLSClientConfig: tlsConfig, ExpectContinueTimeout: 5 * time.Second, } + client := &http.Client{Transport: transport} resolver := docker.NewResolver(docker.ResolverOptions{ - Credentials: func(s string) (string, string, error) { - if i := strings.IndexByte(opt.Auth, ':'); i > 0 { - return opt.Auth[0:i], opt.Auth[i+1:], nil - } - return "", "", nil - }, - PlainHTTP: opt.PlainHTTP, - Client: &http.Client{ - Transport: transport, - }, + Hosts: docker.ConfigureDefaultRegistries( + docker.WithAuthorizer(docker.NewDockerAuthorizer( + docker.WithAuthClient(client), + docker.WithAuthHeader(make(http.Header)), + docker.WithAuthCreds(func(s string) (string, string, error) { + if i := strings.IndexByte(opt.Auth, ':'); i > 0 { + return opt.Auth[0:i], opt.Auth[i+1:], nil + } + return "", "", nil + }), + )), + docker.WithClient(client), + docker.WithPlainHTTP(func(s string) (bool, error) { + if opt.PlainHTTP { + return docker.MatchAllHosts(s) + } else { + return false, nil + } + }), + ), }) engineBase, err := getBuilderEngineBase(ctx, resolver, opt.Ref, opt.TargetRef) if err != nil {