Skip to content

Commit

Permalink
Add implementation for save index job (#2227)
Browse files Browse the repository at this point in the history
* add index save implementation

Signed-off-by: hlts2 <[email protected]>

* make gotest/gen and make format

Signed-off-by: hlts2 <[email protected]>

* fix: sample configuration data

Signed-off-by: hlts2 <[email protected]>

* feat: add save indexing test

Signed-off-by: hlts2 <[email protected]>

* style: format code with Gofumpt and Prettier

This commit fixes the style issues introduced in 84ade79 according to the output
from Gofumpt and Prettier.

Details: #2227

* fix: deleted unnecessary code

Signed-off-by: hlts2 <[email protected]>

* fix: update debug error message

Signed-off-by: hlts2 <[email protected]>

* fix: option name

Signed-off-by: hlts2 <[email protected]>

* fix: test case name

Signed-off-by: hlts2 <[email protected]>

* Update cmd/index/job/save/sample.yaml

Co-authored-by: Kiichiro YUKAWA <[email protected]>

* Update pkg/index/job/save/config/config.go

Co-authored-by: Kiichiro YUKAWA <[email protected]>

* fix: add comment for test case

Signed-off-by: hlts2 <[email protected]>

* fix: fix method naming

Signed-off-by: hlts2 <[email protected]>

* feat: add mock for discoverer client and grpc client

Signed-off-by: hlts2 <[email protected]>

* fix: struct tag for config

Signed-off-by: hlts2 <[email protected]>

* fix: cyclic import for testing

Signed-off-by: hlts2 <[email protected]>

---------

Signed-off-by: hlts2 <[email protected]>
Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
Co-authored-by: Kiichiro YUKAWA <[email protected]>
  • Loading branch information
3 people authored Nov 8, 2023
1 parent cadd72e commit c5cbfc9
Show file tree
Hide file tree
Showing 19 changed files with 2,237 additions and 0 deletions.
78 changes: 78 additions & 0 deletions .github/workflows/dockers-index-save.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#
# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "Build docker image: index-save"
on:
push:
branches:
- main
tags:
- "*.*.*"
- "v*.*.*"
- "*.*.*-*"
- "v*.*.*-*"
paths:
- ".github/actions/docker-build/actions.yaml"
- ".github/workflows/dockers-index-save.yml"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/index/job/save/**"
- "cmd/index/job/save/**"
- "dockers/index/job/save/Dockerfile"
- "versions/GO_VERSION"
pull_request:
paths:
- ".github/actions/docker-build/actions.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-index-save.yml"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/index/job/save/**"
- "cmd/index/job/save/**"
- "dockers/index/job/save/Dockerfile"
- "versions/GO_VERSION"
pull_request_target:
paths:
- ".github/actions/docker-build/actions.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-index-save.yml"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/index/job/save/**"
- "cmd/index/job/save/**"
- "dockers/index/job/save/Dockerfile"
- "versions/GO_VERSION"

jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
with:
target: index-save
secrets: inherit
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ LB_GATEWAY_IMAGE = $(NAME)-lb-gateway
LOADTEST_IMAGE = $(NAME)-loadtest
INDEX_CORRECTION_IMAGE = $(NAME)-index-correction
INDEX_CREATION_IMAGE = $(NAME)-index-creation
INDEX_SAVE_IMAGE = $(NAME)-index-save
MANAGER_INDEX_IMAGE = $(NAME)-manager-index
MAINTAINER = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>"

Expand Down
29 changes: 29 additions & 0 deletions Makefile.d/build.mk
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,35 @@ cmd/index/job/creation/index-creation: \
$(dir $@)main.go
$@ -version

cmd/index/job/save/index-save: \
$(GO_SOURCES_INTERNAL) \
$(PBGOS) \
$(shell find $(ROOTDIR)/cmd/index/job/save -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
$(shell find $(ROOTDIR)/pkg/index/job/save -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
$(eval CGO_ENABLED = 0)
CGO_ENABLED=$(CGO_ENABLED) \
GO111MODULE=on \
GOPRIVATE=$(GOPRIVATE) \
go build \
--ldflags "-w -extldflags=-static \
-X '$(GOPKG)/internal/info.Version=$(VERSION)' \
-X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \
-X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \
-X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \
-X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \
-X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \
-X '$(GOPKG)/internal/info.CGOEnabled=$(CGO_ENABLED)' \
-X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \
-buildid=" \
-mod=readonly \
-modcacherw \
-a \
-tags "osusergo netgo static_build" \
-trimpath \
-o $@ \
$(dir $@)main.go
$@ -version

.PHONY: binary/build/zip
## build all binaries and zip them
binary/build/zip: \
Expand Down
14 changes: 14 additions & 0 deletions Makefile.d/docker.mk
Original file line number Diff line number Diff line change
Expand Up @@ -216,3 +216,17 @@ docker/build/index-creation:
-t $(ORG)/$(INDEX_CREATION_IMAGE):$(TAG) . \
--build-arg MAINTAINER=$(MAINTAINER) \
--build-arg GO_VERSION=$(GO_VERSION)

.PHONY: docker/name/index-save
docker/name/index-save:
@echo "$(ORG)/$(INDEX_SAVE_IMAGE)"

.PHONY: docker/build/index-save
## build index-save image
docker/build/index-save:
$(DOCKER) build \
$(DOCKER_OPTS) \
-f dockers/index/job/save/Dockerfile \
-t $(ORG)/$(INDEX_SAVE_IMAGE):$(TAG) . \
--build-arg MAINTAINER=$(MAINTAINER) \
--build-arg GO_VERSION=$(GO_VERSION)
59 changes: 59 additions & 0 deletions cmd/index/job/save/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
"context"

"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/info"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/runner"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/pkg/index/job/save/config"
"github.com/vdaas/vald/pkg/index/job/save/usecase"
)

const (
maxVersion = "v0.0.10"
minVersion = "v0.0.0"
name = "index save job"
)

func main() {
if err := safety.RecoverFunc(func() error {
return runner.Do(
context.Background(),
runner.WithName(name),
runner.WithVersion(info.Version, maxVersion, minVersion),
runner.WithConfigLoader(func(path string) (interface{}, *config.GlobalConfig, error) {
cfg, err := config.NewConfig(path)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration")
}
return cfg, &cfg.GlobalConfig, nil
}),
runner.WithDaemonInitializer(func(cfg interface{}) (runner.Runner, error) {
c, ok := cfg.(*config.Data)
if !ok {
return nil, errors.ErrInvalidConfig
}
return usecase.New(c)
}),
)
})(); err != nil {
log.Fatal(err, info.Get())
return
}
}
Loading

0 comments on commit c5cbfc9

Please sign in to comment.