Skip to content

Commit

Permalink
refactor docker and change buildkit-syft-scanner reference to ghcr.io
Browse files Browse the repository at this point in the history
Signed-off-by: kpango <[email protected]>
  • Loading branch information
kpango committed Aug 7, 2024
1 parent 3457a5b commit ea18fb4
Show file tree
Hide file tree
Showing 38 changed files with 182 additions and 383 deletions.
67 changes: 2 additions & 65 deletions .gitfiles
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
.github/workflows/dockers-binfmt-image.yaml
.github/workflows/dockers-buildbase-image.yml
.github/workflows/dockers-buildkit-image.yaml
.github/workflows/dockers-buildkit-syft-scanner-image.yaml
.github/workflows/dockers-ci-container-image.yml
.github/workflows/dockers-dev-container-image.yml
.github/workflows/dockers-discoverer-k8s-image.yml
Expand Down Expand Up @@ -292,17 +293,11 @@ assets/test/templates/common/call.tmpl
assets/test/templates/common/fill.tmpl
assets/test/templates/common/function.tmpl
assets/test/templates/common/header.tmpl
assets/test/templates/common/inline.tmpl
assets/test/templates/common/inputs.tmpl
assets/test/templates/common/message.tmpl
assets/test/templates/common/results.tmpl
assets/test/templates/option/call.tmpl
assets/test/templates/option/fill.tmpl
assets/test/templates/option/function.tmpl
assets/test/templates/option/header.tmpl
assets/test/templates/option/inline.tmpl
assets/test/templates/option/inputs.tmpl
assets/test/templates/option/message.tmpl
assets/test/templates/option/results.tmpl
buf.gen.yaml
buf.work.yaml
Expand Down Expand Up @@ -516,7 +511,6 @@ cmd/tools/cli/loadtest/main_test.go
cmd/tools/cli/loadtest/sample.yaml
cmd/tools/cli/vdctl/main.go
cmd/tools/cli/vdctl/main_test.go
design/.gitkeep
design/Vald Architecture Assets.drawio
design/Vald Architecture Dataflow.drawio
design/Vald Architecture Overview.drawio
Expand All @@ -531,6 +525,7 @@ dockers/agent/sidecar/README.md
dockers/binfmt/Dockerfile
dockers/buildbase/Dockerfile
dockers/buildkit/Dockerfile
dockers/buildkit/syft/scanner/Dockerfile
dockers/ci/base/Dockerfile
dockers/ci/base/README.md
dockers/dev/Dockerfile
Expand Down Expand Up @@ -1403,85 +1398,41 @@ internal/worker/worker.go
internal/worker/worker_option.go
internal/worker/worker_option_test.go
internal/worker/worker_test.go
k8s/agent/clusterrole.yaml
k8s/agent/clusterrolebinding.yaml
k8s/agent/daemonset.yaml
k8s/agent/deployment.yaml
k8s/agent/faiss/configmap.yaml
k8s/agent/hpa.yaml
k8s/agent/networkpolicy.yaml
k8s/agent/ngt/configmap.yaml
k8s/agent/pdb.yaml
k8s/agent/priorityclass.yaml
k8s/agent/serviceaccount.yaml
k8s/agent/sidecar/configmap.yaml
k8s/agent/sidecar/svc.yaml
k8s/agent/statefulset.yaml
k8s/agent/svc.yaml
k8s/debug/kind/config.yaml
k8s/discoverer/clusterrole.yaml
k8s/discoverer/clusterrolebinding.yaml
k8s/discoverer/configmap.yaml
k8s/discoverer/daemonset.yaml
k8s/discoverer/deployment.yaml
k8s/discoverer/hpa.yaml
k8s/discoverer/networkpolicy.yaml
k8s/discoverer/pdb.yaml
k8s/discoverer/priorityclass.yaml
k8s/discoverer/serviceaccount.yaml
k8s/discoverer/svc.yaml
k8s/external/minio/deployment.yaml
k8s/external/minio/mb-job.yaml
k8s/external/minio/svc.yaml
k8s/gateway/gateway/filter/configmap.yaml
k8s/gateway/gateway/filter/daemonset.yaml
k8s/gateway/gateway/filter/deployment.yaml
k8s/gateway/gateway/filter/hpa.yaml
k8s/gateway/gateway/filter/networkpolicy.yaml
k8s/gateway/gateway/filter/pdb.yaml
k8s/gateway/gateway/filter/priorityclass.yaml
k8s/gateway/gateway/filter/svc.yaml
k8s/gateway/gateway/ing.yaml
k8s/gateway/gateway/lb/configmap.yaml
k8s/gateway/gateway/lb/daemonset.yaml
k8s/gateway/gateway/lb/deployment.yaml
k8s/gateway/gateway/lb/hpa.yaml
k8s/gateway/gateway/lb/networkpolicy.yaml
k8s/gateway/gateway/lb/pdb.yaml
k8s/gateway/gateway/lb/priorityclass.yaml
k8s/gateway/gateway/lb/svc.yaml
k8s/gateway/gateway/mirror/clusterrole.yaml
k8s/gateway/gateway/mirror/clusterrolebinding.yaml
k8s/gateway/gateway/mirror/configmap.yaml
k8s/gateway/gateway/mirror/daemonset.yaml
k8s/gateway/gateway/mirror/deployment.yaml
k8s/gateway/gateway/mirror/hpa.yaml
k8s/gateway/gateway/mirror/networkpolicy.yaml
k8s/gateway/gateway/mirror/pdb.yaml
k8s/gateway/gateway/mirror/priorityclass.yaml
k8s/gateway/gateway/mirror/serviceaccount.yaml
k8s/gateway/gateway/mirror/svc.yaml
k8s/index/job/correction/configmap.yaml
k8s/index/job/correction/cronjob.yaml
k8s/index/job/correction/networkpolicy.yaml
k8s/index/job/creation/configmap.yaml
k8s/index/job/creation/cronjob.yaml
k8s/index/job/creation/networkpolicy.yaml
k8s/index/job/readreplica/rotate/clusterrole.yaml
k8s/index/job/readreplica/rotate/clusterrolebinding.yaml
k8s/index/job/readreplica/rotate/configmap.yaml
k8s/index/job/readreplica/rotate/networkpolicy.yaml
k8s/index/job/readreplica/rotate/serviceaccount.yaml
k8s/index/job/save/configmap.yaml
k8s/index/job/save/cronjob.yaml
k8s/index/job/save/networkpolicy.yaml
k8s/index/operator/configmap.yaml
k8s/index/operator/deployment.yaml
k8s/index/operator/priorityclass.yaml
k8s/manager/index/configmap.yaml
k8s/manager/index/daemonset.yaml
k8s/manager/index/deployment.yaml
k8s/manager/index/networkpolicy.yaml
k8s/manager/index/pdb.yaml
k8s/manager/index/priorityclass.yaml
k8s/manager/index/svc.yaml
Expand Down Expand Up @@ -1540,12 +1491,6 @@ k8s/operator/helm/operator.yaml
k8s/operator/helm/serviceaccount.yaml
k8s/operator/helm/svc.yaml
k8s/readreplica/configmap.yaml
k8s/readreplica/deployment.yaml
k8s/readreplica/hpa.yaml
k8s/readreplica/networkpolicy.yaml
k8s/readreplica/pvc.yaml
k8s/readreplica/snapshot.yaml
k8s/readreplica/svc.yaml
k8s/tools/benchmark/job/clusterrole.yaml
k8s/tools/benchmark/job/clusterrolebinding.yaml
k8s/tools/benchmark/job/serviceaccount.yaml
Expand Down Expand Up @@ -1960,29 +1905,21 @@ rust/libs/ngt-rs/src/lib.rs
rust/libs/ngt/Cargo.toml
rust/libs/ngt/src/lib.rs
rust/libs/proto/Cargo.toml
rust/libs/proto/src/core.v1.rs
rust/libs/proto/src/core.v1.tonic.rs
rust/libs/proto/src/discoverer.v1.rs
rust/libs/proto/src/discoverer.v1.tonic.rs
rust/libs/proto/src/filter.egress.v1.rs
rust/libs/proto/src/filter.egress.v1.tonic.rs
rust/libs/proto/src/filter.ingress.v1.rs
rust/libs/proto/src/filter.ingress.v1.tonic.rs
rust/libs/proto/src/lib.rs
rust/libs/proto/src/mirror.v1.rs
rust/libs/proto/src/mirror.v1.tonic.rs
rust/libs/proto/src/payload.v1.rs
rust/libs/proto/src/rpc.v1.rs
rust/libs/proto/src/sidecar.v1.rs
rust/libs/proto/src/sidecar.v1.tonic.rs
rust/libs/proto/src/vald.v1.rs
rust/libs/proto/src/vald.v1.tonic.rs
rust/rust-toolchain
rust/rust-toolchain.toml
tests/chaos/chart/.helmignore
tests/chaos/chart/Chart.yaml
tests/chaos/chart/README.md
tests/chaos/chart/templates/NOTES.txt
tests/chaos/chart/templates/_helpers.tpl
tests/chaos/chart/templates/network/bandwidth.yaml
tests/chaos/chart/templates/network/partition.yaml
Expand Down
7 changes: 4 additions & 3 deletions .github/workflows/_docker-image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,9 @@ jobs:
(github.event_name == 'push' &&
github.ref == 'refs/heads/main') ||
(github.event_name == 'push' &&
startsWith( github.ref, 'refs/heads/release/v')) ||
startsWith( github.ref, 'refs/tags/')
startsWith(github.ref, 'refs/heads/release/v')) ||
startsWith(github.ref, 'refs/tags/') ||
(github.event_name == 'schedule')
}}
steps:
- name: Get ref
Expand Down Expand Up @@ -80,7 +81,7 @@ jobs:
driver-opts: |
image=ghcr.io/vdaas/vald/vald-buildkit:nightly
network=host
buildkitd-flags: "--debug --oci-worker-gc=false"
buildkitd-flags: "--debug --oci-worker-gc=false --oci-worker-snapshotter=stargz"
- name: Login to DockerHub
uses: docker/login-action@v3
with:
Expand Down
10 changes: 6 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ TAG ?= latest
CRORG ?= $(ORG)
GHCRORG = ghcr.io/$(REPO)
AGENT_IMAGE = $(NAME)-agent
AGENT_NGT_IMAGE = $(NAME)-agent-ngt
AGENT_FAISS_IMAGE = $(NAME)-agent-faiss
AGENT_SIDECAR_IMAGE = $(NAME)-agent-sidecar
AGENT_NGT_IMAGE = $(AGENT_IMAGE)-ngt
AGENT_FAISS_IMAGE = $(AGENT_IMAGE)-faiss
AGENT_SIDECAR_IMAGE = $(AGENT_IMAGE)-sidecar
BENCHMARK_JOB_IMAGE = $(NAME)-benchmark-job
BENCHMARK_OPERATOR_IMAGE = $(NAME)-benchmark-operator
BINFMT_IMAGE = $(NAME)-binfmt
BUILDBASE_IMAGE = $(NAME)-buildbase
BUILDKIT_IMAGE = $(NAME)-buildkit
BUILDKIT_SYFT_SCANNER_IMAGE = $(NAME)-buildkit-syft-scanner
BUILDKIT_SYFT_SCANNER_IMAGE = $(BUILDKIT_IMAGE)-syft-scanner
CI_CONTAINER_IMAGE = $(NAME)-ci-container
DEV_CONTAINER_IMAGE = $(NAME)-dev-container
DISCOVERER_IMAGE = $(NAME)-discoverer-k8s
Expand All @@ -49,6 +49,8 @@ MIRROR_GATEWAY_IMAGE = $(NAME)-mirror-gateway
READREPLICA_ROTATE_IMAGE = $(NAME)-readreplica-rotate
MAINTAINER = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>"

DEFAULT_BUILDKIT_SYFT_SCANNER_IMAGE = $(GHCRORG)/$(BUILDKIT_SYFT_SCANNER_IMAGE):nightly

VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION)

NGT_REPO = github.com/yahoojapan/NGT
Expand Down
3 changes: 2 additions & 1 deletion Makefile.d/docker.mk
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ ifeq ($(REMOTE),true)
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg RUST_VERSION=$(RUST_VERSION) \
--build-arg MAINTAINER=$(MAINTAINER) \
--attest type=sbom,generator=docker/buildkit-syft-scanner:edge \
--attest type=sbom,generator=$(DEFAULT_BUILDKIT_SYFT_SCANNER_IMAGE) \
--provenance=mode=max \
-t $(CRORG)/$(IMAGE):$(TAG) \
-t $(GHCRORG)/$(IMAGE):$(TAG) \
Expand Down Expand Up @@ -229,6 +229,7 @@ docker/name/buildkit-syft-scanner:
docker/build/buildkit-syft-scanner:
@make DOCKERFILE="$(ROOTDIR)/dockers/buildkit/syft/scanner/Dockerfile" \
IMAGE=$(BUILDKIT_SYFT_SCANNER_IMAGE) \
DEFAULT_BUILDKIT_SYFT_SCANNER_IMAGE="docker/buildkit-syft-scanner:edge" \
docker/build/image

.PHONY: docker/name/ci-container
Expand Down
21 changes: 6 additions & 15 deletions dockers/agent/core/agent/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,13 @@
# limitations under the License.
#

# DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go

# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
ARG UPX_OPTIONS=-9
# skipcq: DOK-DL3026,DOK-DL3007
FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder
ARG MAINTAINER="vdaas.org vald team <[email protected]>"
LABEL maintainer="${MAINTAINER}"

LABEL maintainer="vdaas.org vald team <[email protected]>"
# skipcq: DOK-DL3002
USER root:root

ARG TARGETARCH
ARG TARGETOS
ARG GO_VERSION
Expand All @@ -43,10 +39,9 @@ ENV REPO=vald
ENV RUST_HOME=/usr/loacl/lib/rust
ENV TZ=Etc/UTC
ENV USER=root
ENV RUSTUP_HOME=${RUST_HOME}/rustup
ENV CARGO_HOME=${RUST_HOME}/cargo
ENV RUSTUP_HOME=${RUST_HOME}/rustup
ENV PATH=${CARGO_HOME}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}

WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO}
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008
Expand Down Expand Up @@ -92,12 +87,8 @@ RUN --mount=type=bind,target=.,rw \
&& rm -rf rust/target
# skipcq: DOK-DL3026,DOK-DL3007
FROM gcr.io/distroless/cc-debian12:nonroot
ARG MAINTAINER="vdaas.org vald team <[email protected]>"
LABEL maintainer="${MAINTAINER}"

ENV APP_NAME=agent

COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
LABEL maintainer="vdaas.org vald team <[email protected]>"
COPY --from=builder /usr/bin/agent /usr/bin/agent
# skipcq: DOK-DL3002
USER nonroot:nonroot
ENTRYPOINT ["/usr/bin/agent"]
ENTRYPOINT ["/usr/bin/agent"]
19 changes: 5 additions & 14 deletions dockers/agent/core/faiss/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,13 @@
# limitations under the License.
#

# DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go

# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
ARG UPX_OPTIONS=-9
# skipcq: DOK-DL3026,DOK-DL3007
FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder
ARG MAINTAINER="vdaas.org vald team <[email protected]>"
LABEL maintainer="${MAINTAINER}"

LABEL maintainer="vdaas.org vald team <[email protected]>"
# skipcq: DOK-DL3002
USER root:root

ARG TARGETARCH
ARG TARGETOS
ARG GO_VERSION
Expand All @@ -46,7 +42,6 @@ ENV REPO=vald
ENV TZ=Etc/UTC
ENV USER=root
ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH}

WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008
Expand Down Expand Up @@ -93,13 +88,9 @@ RUN --mount=type=bind,target=.,rw \
&& mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
# skipcq: DOK-DL3026,DOK-DL3007
FROM gcr.io/distroless/static:nonroot
ARG MAINTAINER="vdaas.org vald team <[email protected]>"
LABEL maintainer="${MAINTAINER}"

ENV APP_NAME=faiss

COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
LABEL maintainer="vdaas.org vald team <[email protected]>"
COPY --from=builder /usr/bin/faiss /usr/bin/faiss
COPY cmd/agent/core/faiss/sample.yaml /etc/server/config.yaml
# skipcq: DOK-DL3002
USER nonroot:nonroot
ENTRYPOINT ["/usr/bin/faiss"]
ENTRYPOINT ["/usr/bin/faiss"]
19 changes: 5 additions & 14 deletions dockers/agent/core/ngt/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,13 @@
# limitations under the License.
#

# DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go

# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
ARG UPX_OPTIONS=-9
# skipcq: DOK-DL3026,DOK-DL3007
FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder
ARG MAINTAINER="vdaas.org vald team <[email protected]>"
LABEL maintainer="${MAINTAINER}"

LABEL maintainer="vdaas.org vald team <[email protected]>"
# skipcq: DOK-DL3002
USER root:root

ARG TARGETARCH
ARG TARGETOS
ARG GO_VERSION
Expand All @@ -46,7 +42,6 @@ ENV REPO=vald
ENV TZ=Etc/UTC
ENV USER=root
ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH}

WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008
Expand Down Expand Up @@ -92,13 +87,9 @@ RUN --mount=type=bind,target=.,rw \
&& mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
# skipcq: DOK-DL3026,DOK-DL3007
FROM gcr.io/distroless/static:nonroot
ARG MAINTAINER="vdaas.org vald team <[email protected]>"
LABEL maintainer="${MAINTAINER}"

ENV APP_NAME=ngt

COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
LABEL maintainer="vdaas.org vald team <[email protected]>"
COPY --from=builder /usr/bin/ngt /usr/bin/ngt
COPY cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml
# skipcq: DOK-DL3002
USER nonroot:nonroot
ENTRYPOINT ["/usr/bin/ngt"]
ENTRYPOINT ["/usr/bin/ngt"]
Loading

0 comments on commit ea18fb4

Please sign in to comment.