From 9d074bcf08926e067cbd667ea41209eac28316d7 Mon Sep 17 00:00:00 2001 From: kpango Date: Tue, 9 Jun 2020 00:44:04 +0900 Subject: [PATCH 1/4] add loadtest job and container Signed-off-by: kpango --- .github/labeler.yml | 9 ++ .github/workflows/dockers-loadtest-image.yml | 88 +++++++++++++++++++ Makefile | 1 + Makefile.d/docker.mk | 9 ++ README.md | 6 ++ cmd/{ => tools}/cli/loadtest/main.go | 0 cmd/{ => tools}/cli/loadtest/sample.yaml | 0 cmd/{ => tools}/cli/vdctl/main.go | 0 dockers/base/Dockerfile | 2 +- dockers/tools/cli/loadtest/Dockerfile | 89 ++++++++++++++++++++ k8s/tools/cli/loadtest/configmap.yaml | 85 +++++++++++++++++++ k8s/tools/cli/loadtest/cronjob.yaml | 58 +++++++++++++ k8s/tools/cli/loadtest/job.yaml | 54 ++++++++++++ 13 files changed, 400 insertions(+), 1 deletion(-) create mode 100755 .github/workflows/dockers-loadtest-image.yml rename cmd/{ => tools}/cli/loadtest/main.go (100%) rename cmd/{ => tools}/cli/loadtest/sample.yaml (100%) rename cmd/{ => tools}/cli/vdctl/main.go (100%) create mode 100644 dockers/tools/cli/loadtest/Dockerfile create mode 100644 k8s/tools/cli/loadtest/configmap.yaml create mode 100644 k8s/tools/cli/loadtest/cronjob.yaml create mode 100644 k8s/tools/cli/loadtest/job.yaml diff --git a/.github/labeler.yml b/.github/labeler.yml index 8c1fe4f425..e98f315a01 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -58,6 +58,15 @@ area/meta: - cmd/meta/**/* - pkg/meta/**/* +area/tools/cli/loadtest: + - apis/proto/agent/core/**/* + - apis/grpc/agent/core/**/* + - apis/proto/vald/**/* + - apis/grpc/vald/**/* + - cmd/tools/cli/loadtest/**/* + - pkg/tools/cli/loadtest/**/* + + area/internal: - internal/**/* diff --git a/.github/workflows/dockers-loadtest-image.yml b/.github/workflows/dockers-loadtest-image.yml new file mode 100755 index 0000000000..2f69c4a7a6 --- /dev/null +++ b/.github/workflows/dockers-loadtest-image.yml @@ -0,0 +1,88 @@ +name: 'Build docker image: loadtest' +on: + push: + branches: + - master + tags: + - '*.*.*' + - 'v*.*.*' + - '*.*.*-*' + - 'v*.*.*-*' + paths: + - 'internal/**' + - '!internal/db/**' + - '!internal/k8s/**' + - 'apis/grpc/**' + - 'pkg/tools/cli/loadtest/**' + - 'cmd/tools/cli/loadtest/**' + - 'dockers/base/Dockerfile' + - 'dockers/tools/cli/loadtest/Dockerfile' + - 'versions/GO_VERSION' + pull_request: + paths: + - 'internal/**' + - '!internal/db/**' + - '!internal/k8s/**' + - 'apis/grpc/**' + - 'pkg/tools/cli/loadtest/**' + - 'cmd/tools/cli/loadtest/**' + - 'dockers/base/Dockerfile' + - 'dockers/tools/cli/loadtest/Dockerfile' + - 'versions/GO_VERSION' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 10 + - name: Build the Docker image + run: | + make docker/build/loadtest + - name: login to DockerHub + run: | + echo ${DOCKERHUB_PASS} | docker login --username ${DOCKERHUB_USER} --password-stdin + env: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }} + - name: push to DockerHub (master) + if: github.ref == 'refs/heads/master' + run: | + imagename=`make docker/name/loadtest` + docker tag ${imagename} ${imagename}:nightly + docker push ${imagename}:nightly + - name: push to DockerHub (pull request) + if: github.event_name == 'pull_request' + run: | + imagename=`make docker/name/loadtest` + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + docker tag ${imagename} ${imagename}:pr-${pr_num} + docker push ${imagename}:pr-${pr_num} + - name: push to DockerHub (tags) + if: startsWith( github.ref, 'refs/tags/') + run: | + imagename=`make docker/name/loadtest` + docker push ${imagename}:latest + tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'` + docker tag ${imagename} ${imagename}:${tag_name} + docker push ${imagename}:${tag_name} + docker tag ${imagename} ${imagename}:nightly + docker push ${imagename}:nightly + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v2 + with: + author_name: vald-gateway image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/Makefile b/Makefile index fdd966a1de..21b69dfd2d 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ MANAGER_COMPRESSOR_IMAGE = $(NAME)-manager-compressor MANAGER_INDEX_IMAGE = $(NAME)-manager-index CI_CONTAINER_IMAGE = $(NAME)-ci-container HELM_OPERATOR_IMAGE = $(NAME)-helm-operator +LOADTEST_IMAGE = $(NAME)-loadtest VERSION := $(eval VALD_VERSION := $(shell cat versions/VALD_VERSION))$(VALD_VERSION) diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 124f935dae..3e90b99418 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -149,6 +149,15 @@ docker/build/operator/helm: -t $(REPO)/$(HELM_OPERATOR_IMAGE) . \ --build-arg OPERATOR_SDK_VERSION=$(OPERATOR_SDK_VERSION) +.PHONY: docker/name/loadtest +docker/name/loadtest: + @echo "$(REPO)/$(LOADTEST_IMAGE)" + +.PHONY: docker/build/loadtest +## build loadtest image +docker/build/loadtest: + docker build -f dockers/tools/cli/loadtest/Dockerfile -t $(REPO)/$(LOADTEST_IMAGE) . + .PHONY: dockfmt/install dockfmt/install: $(BINDIR)/dockfmt diff --git a/README.md b/README.md index e9d5452e81..9d1fccbd4b 100755 --- a/README.md +++ b/README.md @@ -189,6 +189,12 @@ make init vdaas/vald-helm-operator + + loadtester + + vdaas/vald-loadtest + + ## Contribution diff --git a/cmd/cli/loadtest/main.go b/cmd/tools/cli/loadtest/main.go similarity index 100% rename from cmd/cli/loadtest/main.go rename to cmd/tools/cli/loadtest/main.go diff --git a/cmd/cli/loadtest/sample.yaml b/cmd/tools/cli/loadtest/sample.yaml similarity index 100% rename from cmd/cli/loadtest/sample.yaml rename to cmd/tools/cli/loadtest/sample.yaml diff --git a/cmd/cli/vdctl/main.go b/cmd/tools/cli/vdctl/main.go similarity index 100% rename from cmd/cli/vdctl/main.go rename to cmd/tools/cli/vdctl/main.go diff --git a/dockers/base/Dockerfile b/dockers/base/Dockerfile index 94250576ab..7f151302b9 100644 --- a/dockers/base/Dockerfile +++ b/dockers/base/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM ubuntu:latest AS builder +FROM ubuntu:devel AS builder ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive diff --git a/dockers/tools/cli/loadtest/Dockerfile b/dockers/tools/cli/loadtest/Dockerfile new file mode 100644 index 0000000000..74fbc01ad2 --- /dev/null +++ b/dockers/tools/cli/loadtest/Dockerfile @@ -0,0 +1,89 @@ +# +# Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM vdaas/vald-base:latest AS builder + +ENV ORG vdaas +ENV REPO vald +ENV APP_NAME loadtest +ENV PKG tools/cli/${APP_NAME} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libhdf5-dev \ + && ldconfig \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal +COPY internal . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/apis/grpc +COPY apis/grpc . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG} +COPY pkg/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +COPY cmd/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +COPY versions/GO_VERSION . +COPY versions/VALD_VERSION . +COPY .git . +RUN GO_VERSION="$(cat GO_VERSION)" \ + && VALD_VERSION="$(cat VALD_VERSION)" \ + && GIT_COMMIT="$(git rev-list -1 HEAD)" \ + && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ + && GOOS="$(go env GOOS)" \ + && GOARCH="$(go env GOARCH)" \ + && CGO_ENABLED=1 \ + && CGO_CXXFLAGS="-g -Ofast -march=native" \ + CGO_FFLAGS="-g -Ofast -march=native" \ + CGO_LDFLAGS="-g -Ofast -march=native" \ + GO111MODULE=on \ + go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static -fPIC -m64 -pthread -fopenmp -std=c++17 -lstdc++ -lm' \ + -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ + -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ + -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ + -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ + -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ + -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ + -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ + -a \ + -tags "cgo netgo" \ + -trimpath \ + -installsuffix "cgo netgo" \ + -o "${APP_NAME}" \ + "cmd/${PKG}/main.go" \ + && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" + +# Start From Scratch For Running Environment +FROM scratch +# Start From Alpine For Debug Environment +# FROM alpine:latest + +ENV APP_NAME loadtest + +# Copy certificates for SSL/TLS +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +# Copy permissions +COPY --from=builder /etc/passwd /etc/passwd +# Copy our static executable +COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} + +ENTRYPOINT ["/go/bin/loadtest"] diff --git a/k8s/tools/cli/loadtest/configmap.yaml b/k8s/tools/cli/loadtest/configmap.yaml new file mode 100644 index 0000000000..379d37a6ae --- /dev/null +++ b/k8s/tools/cli/loadtest/configmap.yaml @@ -0,0 +1,85 @@ +--- +# +# Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: vald-loadtest-config + labels: + app.kubernetes.io/name: vald + app.kubernetes.io/component: loadtest +data: + config.yaml: | + --- + version: v0.0.0 + time_zone: JST + logging: + logger: glg + level: debug + format: json + method: insert + dataset: fashion-mnist + concurrency: 100 + addr: "localhost:8081" + client: + addrs: [] + health_check_duration: 1s + backoff: + backoff_factor: 0 + backoff_time_limit: "" + enable_error_log: false + initial_duration: "" + jitter_limit: "" + maximum_duration: "" + retry_count: 0 + call_option: + max_recv_msg_size: 0 + max_retry_rpc_buffer_size: 0 + max_send_msg_size: 0 + wait_for_ready: true + dial_option: + enable_backoff: false + initial_connection_window_size: 0 + initial_window_size: 0 + insecure: true + keep_alive: + permit_without_stream: false + time: "" + timeout: "" + max_backoff_delay: "" + max_msg_size: 0 + read_buffer_size: 0 + tcp: + dialer: + dual_stack_enabled: true + keep_alive: "" + timeout: "" + dns: + cache_enabled: false + cache_expiration: "" + refresh_duration: "" + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + timeout: "" + write_buffer_size: 0 + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key diff --git a/k8s/tools/cli/loadtest/cronjob.yaml b/k8s/tools/cli/loadtest/cronjob.yaml new file mode 100644 index 0000000000..904bae288e --- /dev/null +++ b/k8s/tools/cli/loadtest/cronjob.yaml @@ -0,0 +1,58 @@ +--- +# +# Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: batch/v1 +kind: Job +metadata: + name: vald-loadtest + labels: + app: vald-loadtest + app.kubernetes.io/name: vald + app.kubernetes.io/component: loadtest +spec: + schedule: "*/1 * * * *" + jobTemplate: + spec: + parallelism: 1 + backoffLimit: 10 + template: + metadata: + app: vald-loadtest + app.kubernetes.io/name: vald + app.kubernetes.io/component: loadtest + spec: + containers: + - name: vald-loadtest + image: "vdaas/vald-meta-redis:nightly" + imagePullPolicy: Always + resources: + limits: + cpu: 300m + memory: 100Mi + requests: + cpu: 100m + memory: 30Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - name: vald-loadtest-config + mountPath: /etc/server/ + dnsPolicy: ClusterFirst + restartPolicy: Never + volumes: + - name: vald-loadtest-config + configMap: + name: vald-loadtest-config diff --git a/k8s/tools/cli/loadtest/job.yaml b/k8s/tools/cli/loadtest/job.yaml new file mode 100644 index 0000000000..f06fffbfa0 --- /dev/null +++ b/k8s/tools/cli/loadtest/job.yaml @@ -0,0 +1,54 @@ +--- +# +# Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: batch/v1 +kind: Job +metadata: + name: vald-loadtest + labels: + app: vald-loadtest + app.kubernetes.io/name: vald + app.kubernetes.io/component: loadtest +spec: + backoffLimit: 5 + template: + metadata: + app: vald-loadtest + app.kubernetes.io/name: vald + app.kubernetes.io/component: loadtest + spec: + containers: + - name: vald-loadtest + image: "vdaas/vald-meta-redis:nightly" + imagePullPolicy: Always + resources: + limits: + cpu: 300m + memory: 100Mi + requests: + cpu: 100m + memory: 30Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - name: vald-loadtest-config + mountPath: /etc/server/ + dnsPolicy: ClusterFirst + restartPolicy: Never + volumes: + - name: vald-loadtest-config + configMap: + name: vald-loadtest-config From 75a0a159b89d618afe7a7c981405fb5293dd4809 Mon Sep 17 00:00:00 2001 From: kpango Date: Tue, 9 Jun 2020 04:10:22 +0900 Subject: [PATCH 2/4] fix Signed-off-by: kpango --- dockers/tools/cli/loadtest/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/tools/cli/loadtest/Dockerfile b/dockers/tools/cli/loadtest/Dockerfile index 74fbc01ad2..090a868616 100644 --- a/dockers/tools/cli/loadtest/Dockerfile +++ b/dockers/tools/cli/loadtest/Dockerfile @@ -56,7 +56,7 @@ RUN GO_VERSION="$(cat GO_VERSION)" \ GO111MODULE=on \ go build \ --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static -fPIC -m64 -pthread -fopenmp -std=c++17 -lstdc++ -lm' \ + -extldflags '-fPIC -m64 -pthread -fopenmp -std=c++17 -lstdc++ -lm' \ -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ From b736d74b79d3f5ab05e8d12062c23ee6ac2a2528 Mon Sep 17 00:00:00 2001 From: kpango Date: Tue, 9 Jun 2020 10:02:46 +0900 Subject: [PATCH 3/4] fix Signed-off-by: kpango --- .github/workflows/dockers-loadtest-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockers-loadtest-image.yml b/.github/workflows/dockers-loadtest-image.yml index 2f69c4a7a6..f246b28c7f 100755 --- a/.github/workflows/dockers-loadtest-image.yml +++ b/.github/workflows/dockers-loadtest-image.yml @@ -80,7 +80,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: 8398a7/action-slack@v2 with: - author_name: vald-gateway image build + author_name: vald-loadtest image build status: ${{ env.WORKFLOW_CONCLUSION }} only_mention_fail: channel env: From 975f5e8f9fad9d9d2b2e94050712037cf0f53e47 Mon Sep 17 00:00:00 2001 From: Yusuke Kato Date: Tue, 9 Jun 2020 10:20:52 +0900 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Rintaro Okamura --- Makefile.d/docker.mk | 2 +- k8s/tools/cli/loadtest/cronjob.yaml | 4 ++-- k8s/tools/cli/loadtest/job.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 3e90b99418..3c8df4ffd0 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -155,7 +155,7 @@ docker/name/loadtest: .PHONY: docker/build/loadtest ## build loadtest image -docker/build/loadtest: +docker/build/loadtest: docker/build/base docker build -f dockers/tools/cli/loadtest/Dockerfile -t $(REPO)/$(LOADTEST_IMAGE) . .PHONY: dockfmt/install diff --git a/k8s/tools/cli/loadtest/cronjob.yaml b/k8s/tools/cli/loadtest/cronjob.yaml index 904bae288e..0457a1a83b 100644 --- a/k8s/tools/cli/loadtest/cronjob.yaml +++ b/k8s/tools/cli/loadtest/cronjob.yaml @@ -15,7 +15,7 @@ # limitations under the License. # apiVersion: batch/v1 -kind: Job +kind: CronJob metadata: name: vald-loadtest labels: @@ -36,7 +36,7 @@ spec: spec: containers: - name: vald-loadtest - image: "vdaas/vald-meta-redis:nightly" + image: "vdaas/vald-loadtest:nightly" imagePullPolicy: Always resources: limits: diff --git a/k8s/tools/cli/loadtest/job.yaml b/k8s/tools/cli/loadtest/job.yaml index f06fffbfa0..6ed652c08a 100644 --- a/k8s/tools/cli/loadtest/job.yaml +++ b/k8s/tools/cli/loadtest/job.yaml @@ -32,7 +32,7 @@ spec: spec: containers: - name: vald-loadtest - image: "vdaas/vald-meta-redis:nightly" + image: "vdaas/vald-loadtest:nightly" imagePullPolicy: Always resources: limits: