From 3157a7ae86b5cfb41f56c2474853e00bed5e5ec6 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Sun, 22 Apr 2018 13:52:59 +0800 Subject: [PATCH] *: Refactor the structure (#65) * cmd: Add CLI Signed-off-by: Ce Gao * scripts: Move the scripts to the directory Signed-off-by: Ce Gao * manager: Refactor Signed-off-by: Ce Gao * mock: Refactor Signed-off-by: Ce Gao * earlystopping: Refactor Signed-off-by: Ce Gao * build.sh: Fix Signed-off-by: Ce Gao * kubernetes: Remove Signed-off-by: Ce Gao * suggestion: Refactor Signed-off-by: Ce Gao * examples: Rename conf to examples Signed-off-by: Ce Gao * api: Refactor Signed-off-by: Ce Gao * *: Fix Signed-off-by: Ce Gao * build.sh: Remove comments Signed-off-by: Ce Gao --- README.md | 4 +- build.sh | 12 ----- {cli => cmd/cli}/Dockerfile | 4 +- {cli => cmd/cli}/main.go | 2 +- .../earlystopping}/medianstopping/Dockerfile | 4 +- .../earlystopping}/medianstopping/main.go | 4 +- cmd/manager/Dockerfile | 12 +++++ {manager => cmd/manager}/main.go | 17 +++---- {manager => cmd/manager}/main_test.go | 8 ++-- {manager => cmd/manager}/wrap.sh | 0 .../bayesianoptimization/Dockerfile | 2 +- .../bayesianoptimization/README.md | 0 .../bayesianoptimization/__init__.py | 0 .../suggestion}/bayesianoptimization/main.py | 7 +-- .../bayesianoptimization/requirements.txt | 0 .../suggestion}/grid/Dockerfile | 4 +- {suggestion => cmd/suggestion}/grid/main.go | 4 +- .../suggestion}/hyperband/Dockerfile | 4 +- .../suggestion}/hyperband/main.go | 4 +- .../suggestion}/random/Dockerfile | 4 +- {suggestion => cmd/suggestion}/random/main.go | 4 +- deploy.sh | 11 ----- docs/developer-guide.md | 2 +- docs/getting-start.md | 6 +-- {conf => examples}/grid.yml | 0 {conf => examples}/hyperband.yml | 0 {conf => examples}/opennmt.yml | 0 {conf => examples}/random-cpu.yml | 0 {conf => examples}/random-pv.yml | 0 {conf => examples}/random.yml | 0 {conf => examples}/tf-nmt.yml | 0 manager/Dockerfile | 12 ----- {api => pkg/api}/Makefile | 0 {api => pkg/api}/api.pb.go | 0 {api => pkg/api}/api.proto | 0 {api => pkg/api}/build.sh | 0 {api => pkg/api}/python/api_pb2.py | 0 {api => pkg/api}/python/api_pb2_grpc.py | 2 +- {db => pkg/db}/db_init.go | 0 {db => pkg/db}/interface.go | 2 +- {db => pkg/db}/interface_test.go | 2 +- {db => pkg/db}/test/test.go | 2 +- .../earlystopping}/medianstopping.go | 4 +- {earlystopping => pkg/earlystopping}/types.go | 2 +- .../manager}/modelstore/modeldb.go | 4 +- .../modeldb/GoUnusedProtection__.go | 0 .../modelstore/modeldb/ModelDB-consts.go | 0 .../manager}/modelstore/modeldb/ModelDB.go | 0 .../model_d_b_service-remote.go | 2 +- .../manager}/modelstore/modelstore.go | 2 +- .../manifest_template/deployment.yaml | 0 .../manifest_template/ingress.yaml | 0 .../manifest_template/service.yaml | 0 .../visualise/tensorboard/tensorboardIF.go | 2 +- .../manager}/worker_interface/dlk/dlk.go | 6 +-- .../worker_interface/kubernetes/kubernetes.go | 19 ++++---- .../nvdocker/nvidia-docker.go | 6 +-- .../worker_interface/worker_interface.go | 4 +- {mock => pkg/mock}/api/manager.go | 4 +- {mock => pkg/mock}/api/suggestion.go | 4 +- {mock => pkg/mock}/db/db.go | 6 +-- {mock => pkg/mock}/modelstore/modelstore.go | 4 +- {mock => pkg/mock}/worker/worker.go | 4 +- .../suggestion}/bayesian_service.py | 8 ++-- .../bayesianoptimization/src/__init__.py | 0 .../src/acquisition_func/__init__.py | 0 .../src/acquisition_func/acquisition_func.py | 2 +- .../src/algorithm_manager.py | 2 +- .../src/bayesian_optimization_algorithm.py | 2 +- .../src/global_optimizer/__init__.py | 0 .../src/global_optimizer/global_optimizer.py | 2 +- .../bayesianoptimization/src/model/Model.py | 0 .../src/model/__init__.py | 0 .../suggestion}/grid_service.go | 2 +- .../suggestion}/hyperband_service.go | 4 +- .../suggestion}/random_service.go | 2 +- {suggestion => pkg/suggestion}/test_client.py | 8 ++-- {suggestion => pkg/suggestion}/test_func.py | 0 {suggestion => pkg/suggestion}/types.go | 2 +- {suggestion => pkg/suggestion}/types.py | 0 scripts/build.sh | 48 +++++++++++++++++++ scripts/deploy.sh | 32 +++++++++++++ scripts/mockgen.sh | 10 ++-- test/scripts/build.sh | 28 +++++------ test/scripts/run-tests.sh | 2 +- 85 files changed, 214 insertions(+), 152 deletions(-) delete mode 100755 build.sh rename {cli => cmd/cli}/Dockerfile (55%) rename {cli => cmd/cli}/main.go (98%) rename {earlystopping => cmd/earlystopping}/medianstopping/Dockerfile (53%) rename {earlystopping => cmd/earlystopping}/medianstopping/main.go (87%) create mode 100644 cmd/manager/Dockerfile rename {manager => cmd/manager}/main.go (96%) rename {manager => cmd/manager}/main_test.go (94%) rename {manager => cmd/manager}/wrap.sh (100%) rename {suggestion => cmd/suggestion}/bayesianoptimization/Dockerfile (70%) rename {suggestion => cmd/suggestion}/bayesianoptimization/README.md (100%) rename {suggestion => cmd/suggestion}/bayesianoptimization/__init__.py (100%) rename {suggestion => cmd/suggestion}/bayesianoptimization/main.py (73%) rename {suggestion => cmd/suggestion}/bayesianoptimization/requirements.txt (100%) rename {suggestion => cmd/suggestion}/grid/Dockerfile (55%) rename {suggestion => cmd/suggestion}/grid/main.go (87%) rename {suggestion => cmd/suggestion}/hyperband/Dockerfile (55%) rename {suggestion => cmd/suggestion}/hyperband/main.go (87%) rename {suggestion => cmd/suggestion}/random/Dockerfile (55%) rename {suggestion => cmd/suggestion}/random/main.go (86%) delete mode 100755 deploy.sh rename {conf => examples}/grid.yml (100%) rename {conf => examples}/hyperband.yml (100%) rename {conf => examples}/opennmt.yml (100%) rename {conf => examples}/random-cpu.yml (100%) rename {conf => examples}/random-pv.yml (100%) rename {conf => examples}/random.yml (100%) rename {conf => examples}/tf-nmt.yml (100%) delete mode 100644 manager/Dockerfile rename {api => pkg/api}/Makefile (100%) rename {api => pkg/api}/api.pb.go (100%) rename {api => pkg/api}/api.proto (100%) rename {api => pkg/api}/build.sh (100%) rename {api => pkg/api}/python/api_pb2.py (100%) rename {api => pkg/api}/python/api_pb2_grpc.py (99%) rename {db => pkg/db}/db_init.go (100%) rename {db => pkg/db}/interface.go (99%) rename {db => pkg/db}/interface_test.go (98%) rename {db => pkg/db}/test/test.go (85%) rename {earlystopping => pkg/earlystopping}/medianstopping.go (98%) rename {earlystopping => pkg/earlystopping}/types.go (93%) rename {manager => pkg/manager}/modelstore/modeldb.go (98%) rename {manager => pkg/manager}/modelstore/modeldb/GoUnusedProtection__.go (100%) rename {manager => pkg/manager}/modelstore/modeldb/ModelDB-consts.go (100%) rename {manager => pkg/manager}/modelstore/modeldb/ModelDB.go (100%) rename {manager => pkg/manager}/modelstore/modeldb/model_d_b_service-remote/model_d_b_service-remote.go (99%) rename {manager => pkg/manager}/modelstore/modelstore.go (89%) rename {manager => pkg/manager}/visualise/tensorboard/manifest_template/deployment.yaml (100%) rename {manager => pkg/manager}/visualise/tensorboard/manifest_template/ingress.yaml (100%) rename {manager => pkg/manager}/visualise/tensorboard/manifest_template/service.yaml (100%) rename {manager => pkg/manager}/visualise/tensorboard/tensorboardIF.go (98%) rename {manager => pkg/manager}/worker_interface/dlk/dlk.go (98%) rename {manager => pkg/manager}/worker_interface/kubernetes/kubernetes.go (97%) rename {manager => pkg/manager}/worker_interface/nvdocker/nvidia-docker.go (98%) rename {manager => pkg/manager}/worker_interface/worker_interface.go (96%) rename {mock => pkg/mock}/api/manager.go (99%) rename {mock => pkg/mock}/api/suggestion.go (96%) rename {mock => pkg/mock}/db/db.go (98%) rename {mock => pkg/mock}/modelstore/modelstore.go (96%) rename {mock => pkg/mock}/worker/worker.go (97%) rename {suggestion => pkg/suggestion}/bayesian_service.py (95%) rename {suggestion => pkg/suggestion}/bayesianoptimization/src/__init__.py (100%) rename {suggestion => pkg/suggestion}/bayesianoptimization/src/acquisition_func/__init__.py (100%) rename {suggestion => pkg/suggestion}/bayesianoptimization/src/acquisition_func/acquisition_func.py (95%) rename {suggestion => pkg/suggestion}/bayesianoptimization/src/algorithm_manager.py (99%) rename {suggestion => pkg/suggestion}/bayesianoptimization/src/bayesian_optimization_algorithm.py (95%) rename {suggestion => pkg/suggestion}/bayesianoptimization/src/global_optimizer/__init__.py (100%) rename {suggestion => pkg/suggestion}/bayesianoptimization/src/global_optimizer/global_optimizer.py (98%) rename {suggestion => pkg/suggestion}/bayesianoptimization/src/model/Model.py (100%) rename {suggestion => pkg/suggestion}/bayesianoptimization/src/model/__init__.py (100%) rename {suggestion => pkg/suggestion}/grid_service.go (99%) rename {suggestion => pkg/suggestion}/hyperband_service.go (99%) rename {suggestion => pkg/suggestion}/random_service.go (99%) rename {suggestion => pkg/suggestion}/test_client.py (96%) rename {suggestion => pkg/suggestion}/test_func.py (100%) rename {suggestion => pkg/suggestion}/types.go (92%) rename {suggestion => pkg/suggestion}/types.py (100%) create mode 100755 scripts/build.sh create mode 100755 scripts/deploy.sh diff --git a/README.md b/README.md index b9000e12653..191503cb519 100644 --- a/README.md +++ b/README.md @@ -95,10 +95,10 @@ Katib provides a Web UI based on ModelDB(https://github.com/mitdbg/modeldb). The In addition to TensorFlow, other deep learning frameworks (e.g. PyTorch, MXNet) support TensorBoard format logging. Katib integrates with TensorBoard easily. To use TensorBoard from Katib, we define a persistent volume claim and set the mount config for the Study. Katib searches each trial log in `{pvc mount path}/logs/{Study ID}/{Trial ID}`. `{{STUDY_ID}}` and `{{TRIAL_ID}}` in the Studyconfig file are replaced the corresponding value when creating each job. -See example `conf/tf-nmt.yml` which is a config for parameter tuning of [tensorflow/nmt](https://github.com/tensorflow/nmt). +See example `examples/tf-nmt.yml` which is a config for parameter tuning of [tensorflow/nmt](https://github.com/tensorflow/nmt). ```bash -./katib-cli -s gpu-node2:30678 -f ../conf/tf-nmt.yml Createstudy +./katib-cli -s gpu-node2:30678 -f ../examples/tf-nmt.yml Createstudy 2018/04/03 05:52:11 connecting gpu-node2:30678 2018/04/03 05:52:11 study conf{tf-nmt root MINIMIZE 0 configs: > configs: > configs: > configs: > configs: > configs: > configs: > [] random median [name:"SuggestionNum" value:"10" name:"MaxParallel" value:"6" ] [] test_ppl [ppl bleu_dev bleu_test] yujioshima/tf-nmt:latest-gpu [python -m nmt.nmt --src=vi --tgt=en --out_dir=/nfs-mnt/logs/{{STUDY_ID}}_{{TRIAL_ID}} --vocab_prefix=/nfs-mnt/learndatas/wmt15_en_vi/vocab --train_prefix=/nfs-mnt/learndatas/wmt15_en_vi/train --dev_prefix=/nfs-mnt/learndatas/wmt15_en_vi/tst2012 --test_prefix=/nfs-mnt/learndatas/wmt15_en_vi/tst2013 --attention_architecture=standard --attention=normed_bahdanau --batch_size=128 --colocate_gradients_with_ops=true --eos= --forget_bias=1.0 --init_weight=0.1 --learning_rate=1.0 --max_gradient_norm=5.0 --metrics=bleu --share_vocab=false --num_buckets=5 --optimizer=sgd --sos= --steps_per_stats=100 --time_major=true --unit_type=lstm --src_max_len=50 --tgt_max_len=50 --infer_batch_size=32] 1 default-scheduler pvc:"nfs" path:"/nfs-mnt" } 2018/04/03 05:52:11 req Createstudy diff --git a/build.sh b/build.sh deleted file mode 100755 index 924326c725b..00000000000 --- a/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#/bin/bash -set -x -set -e -PREFIX="katib/" -docker build -t ${PREFIX}vizier-core -f manager/Dockerfile . -docker build -t ${PREFIX}suggestion-random -f suggestion/random/Dockerfile . -docker build -t ${PREFIX}suggestion-grid -f suggestion/grid/Dockerfile . -docker build -t ${PREFIX}suggestion-hyperband -f suggestion/hyperband/Dockerfile . -docker build -t ${PREFIX}earlystopping-medianstopping -f earlystopping/medianstopping/Dockerfile . -docker build -t ${PREFIX}dlk-manager -f dlk/Dockerfile . -docker build -t ${PREFIX}katib-frontend -f modeldb/Dockerfile . -docker build -t ${PREFIX}katib-cli -f cli/Dockerfile . diff --git a/cli/Dockerfile b/cmd/cli/Dockerfile similarity index 55% rename from cli/Dockerfile rename to cmd/cli/Dockerfile index 4d8c88473d3..cec27d10aa6 100644 --- a/cli/Dockerfile +++ b/cmd/cli/Dockerfile @@ -1,9 +1,9 @@ FROM golang:alpine AS build-env # The GOPATH in the image is /go. ADD . /go/src/github.com/kubeflow/katib -WORKDIR /go/src/github.com/kubeflow/katib/cli +WORKDIR /go/src/github.com/kubeflow/katib/cmd/cli RUN go build -o katib-cli FROM alpine:3.7 WORKDIR /app -COPY --from=build-env /go/src/github.com/kubeflow/katib/cli/katib-cli /app/ +COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/cli/katib-cli /app/ diff --git a/cli/main.go b/cmd/cli/main.go similarity index 98% rename from cli/main.go rename to cmd/cli/main.go index 66640b29d87..3fd7df5e1fa 100644 --- a/cli/main.go +++ b/cmd/cli/main.go @@ -12,7 +12,7 @@ import ( "google.golang.org/grpc" - pb "github.com/kubeflow/katib/api" + pb "github.com/kubeflow/katib/pkg/api" ) var server = flag.String("s", "127.0.0.1:6789", "server address") diff --git a/earlystopping/medianstopping/Dockerfile b/cmd/earlystopping/medianstopping/Dockerfile similarity index 53% rename from earlystopping/medianstopping/Dockerfile rename to cmd/earlystopping/medianstopping/Dockerfile index 372254494ed..dba336f2678 100644 --- a/earlystopping/medianstopping/Dockerfile +++ b/cmd/earlystopping/medianstopping/Dockerfile @@ -1,10 +1,10 @@ FROM golang:alpine AS build-env # The GOPATH in the image is /go. ADD . /go/src/github.com/kubeflow/katib -WORKDIR /go/src/github.com/kubeflow/katib/earlystopping/medianstopping +WORKDIR /go/src/github.com/kubeflow/katib/cmd/earlystopping/medianstopping RUN go build -o medianstopping FROM alpine:3.7 WORKDIR /app -COPY --from=build-env /go/src/github.com/kubeflow/katib/earlystopping/medianstopping /app/ +COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/earlystopping/medianstopping /app/ ENTRYPOINT ["./medianstopping"] diff --git a/earlystopping/medianstopping/main.go b/cmd/earlystopping/medianstopping/main.go similarity index 87% rename from earlystopping/medianstopping/main.go rename to cmd/earlystopping/medianstopping/main.go index a3da6c333a9..ac21f0b339c 100644 --- a/earlystopping/medianstopping/main.go +++ b/cmd/earlystopping/medianstopping/main.go @@ -7,8 +7,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/reflection" - pb "github.com/kubeflow/katib/api" - "github.com/kubeflow/katib/earlystopping" + pb "github.com/kubeflow/katib/pkg/api" + "github.com/kubeflow/katib/pkg/earlystopping" ) func main() { diff --git a/cmd/manager/Dockerfile b/cmd/manager/Dockerfile new file mode 100644 index 00000000000..1085552cb50 --- /dev/null +++ b/cmd/manager/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:alpine AS build-env +# The GOPATH in the image is /go. +ADD . /go/src/github.com/kubeflow/katib +WORKDIR /go/src/github.com/kubeflow/katib/cmd/manager +RUN go build -o vizier-manager + +FROM alpine:3.7 +WORKDIR /app +COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager/vizier-manager /app/ +COPY --from=build-env /go/src/github.com/kubeflow/katib/pkg/manager/visualise / +ENTRYPOINT ["./vizier-manager"] +CMD ["-w", "dlk"] diff --git a/manager/main.go b/cmd/manager/main.go similarity index 96% rename from manager/main.go rename to cmd/manager/main.go index b8342f2d08e..46f9ac52750 100644 --- a/manager/main.go +++ b/cmd/manager/main.go @@ -9,14 +9,14 @@ import ( "strconv" "time" - pb "github.com/kubeflow/katib/api" - kdb "github.com/kubeflow/katib/db" - "github.com/kubeflow/katib/manager/modelstore" - tbif "github.com/kubeflow/katib/manager/visualise/tensorboard" - "github.com/kubeflow/katib/manager/worker_interface" - dlkwif "github.com/kubeflow/katib/manager/worker_interface/dlk" - k8swif "github.com/kubeflow/katib/manager/worker_interface/kubernetes" - nvdwif "github.com/kubeflow/katib/manager/worker_interface/nvdocker" + pb "github.com/kubeflow/katib/pkg/api" + kdb "github.com/kubeflow/katib/pkg/db" + "github.com/kubeflow/katib/pkg/manager/modelstore" + tbif "github.com/kubeflow/katib/pkg/manager/visualise/tensorboard" + "github.com/kubeflow/katib/pkg/manager/worker_interface" + dlkwif "github.com/kubeflow/katib/pkg/manager/worker_interface/dlk" + k8swif "github.com/kubeflow/katib/pkg/manager/worker_interface/kubernetes" + nvdwif "github.com/kubeflow/katib/pkg/manager/worker_interface/nvdocker" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -394,6 +394,7 @@ func main() { switch *workerType { case "kubernetes": log.Printf("Worker: kubernetes\n") + // Notice: Missing in the repo. kc, err := clientcmd.BuildConfigFromFlags("", "/conf/kubeconfig") if err != nil { log.Fatal(err) diff --git a/manager/main_test.go b/cmd/manager/main_test.go similarity index 94% rename from manager/main_test.go rename to cmd/manager/main_test.go index 6f5717f47b6..38b6963d333 100644 --- a/manager/main_test.go +++ b/cmd/manager/main_test.go @@ -6,10 +6,10 @@ import ( "github.com/golang/mock/gomock" - api "github.com/kubeflow/katib/api" - mockdb "github.com/kubeflow/katib/mock/db" - mockmodelstore "github.com/kubeflow/katib/mock/modelstore" - mockworker "github.com/kubeflow/katib/mock/worker" + api "github.com/kubeflow/katib/pkg/api" + mockdb "github.com/kubeflow/katib/pkg/mock/db" + mockmodelstore "github.com/kubeflow/katib/pkg/mock/modelstore" + mockworker "github.com/kubeflow/katib/pkg/mock/worker" ) func TestCreateStudy(t *testing.T) { diff --git a/manager/wrap.sh b/cmd/manager/wrap.sh similarity index 100% rename from manager/wrap.sh rename to cmd/manager/wrap.sh diff --git a/suggestion/bayesianoptimization/Dockerfile b/cmd/suggestion/bayesianoptimization/Dockerfile similarity index 70% rename from suggestion/bayesianoptimization/Dockerfile rename to cmd/suggestion/bayesianoptimization/Dockerfile index 58037727901..861be845fd7 100644 --- a/suggestion/bayesianoptimization/Dockerfile +++ b/cmd/suggestion/bayesianoptimization/Dockerfile @@ -1,7 +1,7 @@ FROM python:3 ADD . /usr/src/app/github.com/kubeflow/katib -WORKDIR /usr/src/app/github.com/kubeflow/katib/suggestion/bayesianoptimization +WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/suggestion/bayesianoptimization RUN pip install --no-cache-dir -r requirements.txt ENV PYTHONPATH /usr/src/app/github.com/kubeflow/katib diff --git a/suggestion/bayesianoptimization/README.md b/cmd/suggestion/bayesianoptimization/README.md similarity index 100% rename from suggestion/bayesianoptimization/README.md rename to cmd/suggestion/bayesianoptimization/README.md diff --git a/suggestion/bayesianoptimization/__init__.py b/cmd/suggestion/bayesianoptimization/__init__.py similarity index 100% rename from suggestion/bayesianoptimization/__init__.py rename to cmd/suggestion/bayesianoptimization/__init__.py diff --git a/suggestion/bayesianoptimization/main.py b/cmd/suggestion/bayesianoptimization/main.py similarity index 73% rename from suggestion/bayesianoptimization/main.py rename to cmd/suggestion/bayesianoptimization/main.py index 79d666d5a03..1a6463d7925 100644 --- a/suggestion/bayesianoptimization/main.py +++ b/cmd/suggestion/bayesianoptimization/main.py @@ -3,9 +3,9 @@ import time -from api.python import api_pb2_grpc -from suggestion.bayesian_service import BayesianService -from suggestion.types import DEFAULT_PORT +from pkg.api.python import api_pb2_grpc +from pkg.suggestion.bayesian_service import BayesianService +from pkg.suggestion.types import DEFAULT_PORT _ONE_DAY_IN_SECONDS = 60 * 60 * 24 @@ -14,6 +14,7 @@ def serve(): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) api_pb2_grpc.add_SuggestionServicer_to_server(BayesianService(), server) server.add_insecure_port(DEFAULT_PORT) + print("Listening...") server.start() try: while True: diff --git a/suggestion/bayesianoptimization/requirements.txt b/cmd/suggestion/bayesianoptimization/requirements.txt similarity index 100% rename from suggestion/bayesianoptimization/requirements.txt rename to cmd/suggestion/bayesianoptimization/requirements.txt diff --git a/suggestion/grid/Dockerfile b/cmd/suggestion/grid/Dockerfile similarity index 55% rename from suggestion/grid/Dockerfile rename to cmd/suggestion/grid/Dockerfile index 671e43fed2a..5dcc2882a37 100644 --- a/suggestion/grid/Dockerfile +++ b/cmd/suggestion/grid/Dockerfile @@ -1,10 +1,10 @@ FROM golang:alpine AS build-env # The GOPATH in the image is /go. ADD . /go/src/github.com/kubeflow/katib -WORKDIR /go/src/github.com/kubeflow/katib/suggestion/grid +WORKDIR /go/src/github.com/kubeflow/katib/cmd/suggestion/grid RUN go build -o grid FROM alpine:3.7 WORKDIR /app -COPY --from=build-env /go/src/github.com/kubeflow/katib/suggestion/grid /app/ +COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/suggestion/grid /app/ ENTRYPOINT ["./grid"] diff --git a/suggestion/grid/main.go b/cmd/suggestion/grid/main.go similarity index 87% rename from suggestion/grid/main.go rename to cmd/suggestion/grid/main.go index d66452bf05e..6acadfbde83 100644 --- a/suggestion/grid/main.go +++ b/cmd/suggestion/grid/main.go @@ -7,8 +7,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/reflection" - pb "github.com/kubeflow/katib/api" - "github.com/kubeflow/katib/suggestion" + pb "github.com/kubeflow/katib/pkg/api" + "github.com/kubeflow/katib/pkg/suggestion" ) func main() { diff --git a/suggestion/hyperband/Dockerfile b/cmd/suggestion/hyperband/Dockerfile similarity index 55% rename from suggestion/hyperband/Dockerfile rename to cmd/suggestion/hyperband/Dockerfile index ae8f415e097..5c264b64731 100644 --- a/suggestion/hyperband/Dockerfile +++ b/cmd/suggestion/hyperband/Dockerfile @@ -1,10 +1,10 @@ FROM golang:alpine AS build-env # The GOPATH in the image is /go. ADD . /go/src/github.com/kubeflow/katib -WORKDIR /go/src/github.com/kubeflow/katib/suggestion/hyperband +WORKDIR /go/src/github.com/kubeflow/katib/cmd/suggestion/hyperband RUN go build -o hyperband FROM alpine:3.7 WORKDIR /app -COPY --from=build-env /go/src/github.com/kubeflow/katib/suggestion/hyperband /app/ +COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/suggestion/hyperband /app/ ENTRYPOINT ["./hyperband"] diff --git a/suggestion/hyperband/main.go b/cmd/suggestion/hyperband/main.go similarity index 87% rename from suggestion/hyperband/main.go rename to cmd/suggestion/hyperband/main.go index 26312dddc43..5a1f04ba34c 100644 --- a/suggestion/hyperband/main.go +++ b/cmd/suggestion/hyperband/main.go @@ -7,8 +7,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/reflection" - pb "github.com/kubeflow/katib/api" - "github.com/kubeflow/katib/suggestion" + pb "github.com/kubeflow/katib/pkg/api" + "github.com/kubeflow/katib/pkg/suggestion" ) func main() { diff --git a/suggestion/random/Dockerfile b/cmd/suggestion/random/Dockerfile similarity index 55% rename from suggestion/random/Dockerfile rename to cmd/suggestion/random/Dockerfile index c59c69fd65c..551c86f351d 100644 --- a/suggestion/random/Dockerfile +++ b/cmd/suggestion/random/Dockerfile @@ -1,10 +1,10 @@ FROM golang:alpine AS build-env # The GOPATH in the image is /go. ADD . /go/src/github.com/kubeflow/katib -WORKDIR /go/src/github.com/kubeflow/katib/suggestion/random +WORKDIR /go/src/github.com/kubeflow/katib/cmd/suggestion/random RUN go build -o random FROM alpine:3.7 WORKDIR /app -COPY --from=build-env /go/src/github.com/kubeflow/katib/suggestion/random /app/ +COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/suggestion/random /app/ ENTRYPOINT ["./random"] diff --git a/suggestion/random/main.go b/cmd/suggestion/random/main.go similarity index 86% rename from suggestion/random/main.go rename to cmd/suggestion/random/main.go index c653b0f0110..9f3320ad824 100644 --- a/suggestion/random/main.go +++ b/cmd/suggestion/random/main.go @@ -7,8 +7,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/reflection" - pb "github.com/kubeflow/katib/api" - "github.com/kubeflow/katib/suggestion" + pb "github.com/kubeflow/katib/pkg/api" + "github.com/kubeflow/katib/pkg/suggestion" ) func main() { diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 5637a101426..00000000000 --- a/deploy.sh +++ /dev/null @@ -1,11 +0,0 @@ -#/bin/bash -set -x -set -e -kubectl apply -f manifests/0-namespace.yaml -kubectl apply -f manifests/modeldb/db -kubectl apply -f manifests/modeldb/backend -kubectl apply -f manifests/modeldb/frontend -kubectl apply -f manifests/dlk -kubectl apply -f manifests/vizier/db -kubectl apply -f manifests/vizier/core -kubectl apply -f manifests/vizier/suggestion/random diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 9f5ec1f26a2..5461ba69227 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -9,7 +9,7 @@ You can build all images from source. ```bash -./build.sh +./scripts/build.sh ``` ## Implement new suggestion algorithm diff --git a/docs/getting-start.md b/docs/getting-start.md index 8454f7c2611..fb792feb20e 100644 --- a/docs/getting-start.md +++ b/docs/getting-start.md @@ -21,7 +21,7 @@ Kubernetes manifests are in `manifests` directory. Set the environment of your cluster(Ingress, Persistent Volumes). ```bash -$ ./deploy.sh +$ ./scripts/deploy.sh ``` ## Use CLI @@ -50,7 +50,7 @@ StudyID Name Owner RunningTrial CompletedTrial Try Createstudy. Study will be created and start hyperparameter search. ```bash -$ katib-cli -s gpu-node2:30678 -f ../conf/random.yml Createstudy +$ katib-cli -s gpu-node2:30678 -f ../examples/random.yml Createstudy 2018/04/03 05:16:37 connecting gpu-node2:30678 2018/04/03 05:16:37 study conf{cifer10 root MAXIMIZE 0 configs: > configs: > configs: > configs: > configs: > [] random median [name:"SuggestionNum" value:"2" name:"MaxParallel" value:"2" ] [] Validation-accuracy [accuracy] mxnet/python:gpu [python /mxnet/example/image-classification/train_cifar10.py --batch-size=512 --gpus=0,1] 2 default-scheduler } 2018/04/03 05:16:37 req Createstudy @@ -215,7 +215,7 @@ parameterconfigs: ``` ```bash -$ katib-cli -s gpu-node2:30678 -f ../conf/random-pv.yml Createstudy +$ katib-cli -s gpu-node2:30678 -f ../examples/random-pv.yml Createstudy 2018/04/03 05:49:47 connecting gpu-node2:30678 2018/04/03 05:49:47 study conf{cifer10-pv-test root MAXIMIZE 0 configs: > configs: > configs: > configs: > configs: > [] random median [name:"SuggestionNum" value:"2" name:"MaxParallel" value:"2" ] [] Validation-accuracy [accuracy] mxnet/python:gpu [python /mxnet/example/image-classification/train_cifar10.py --batch-size=512 --gpus=0,1] 2 default-scheduler pvc:"nfs" path:"/nfs-mnt" } 2018/04/03 05:49:47 req Createstudy diff --git a/conf/grid.yml b/examples/grid.yml similarity index 100% rename from conf/grid.yml rename to examples/grid.yml diff --git a/conf/hyperband.yml b/examples/hyperband.yml similarity index 100% rename from conf/hyperband.yml rename to examples/hyperband.yml diff --git a/conf/opennmt.yml b/examples/opennmt.yml similarity index 100% rename from conf/opennmt.yml rename to examples/opennmt.yml diff --git a/conf/random-cpu.yml b/examples/random-cpu.yml similarity index 100% rename from conf/random-cpu.yml rename to examples/random-cpu.yml diff --git a/conf/random-pv.yml b/examples/random-pv.yml similarity index 100% rename from conf/random-pv.yml rename to examples/random-pv.yml diff --git a/conf/random.yml b/examples/random.yml similarity index 100% rename from conf/random.yml rename to examples/random.yml diff --git a/conf/tf-nmt.yml b/examples/tf-nmt.yml similarity index 100% rename from conf/tf-nmt.yml rename to examples/tf-nmt.yml diff --git a/manager/Dockerfile b/manager/Dockerfile deleted file mode 100644 index 23fe2e0ed15..00000000000 --- a/manager/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM golang:alpine AS build-env -# The GOPATH in the image is /go. -ADD . /go/src/github.com/kubeflow/katib -WORKDIR /go/src/github.com/kubeflow/katib/manager -RUN go build -o vizier-manager - -FROM alpine:3.7 -WORKDIR /app -COPY --from=build-env /go/src/github.com/kubeflow/katib/manager/vizier-manager /app/ -COPY --from=build-env /go/src/github.com/kubeflow/katib/manager/visualise / -ENTRYPOINT ["./vizier-manager"] -CMD ["-w", "dlk"] diff --git a/api/Makefile b/pkg/api/Makefile similarity index 100% rename from api/Makefile rename to pkg/api/Makefile diff --git a/api/api.pb.go b/pkg/api/api.pb.go similarity index 100% rename from api/api.pb.go rename to pkg/api/api.pb.go diff --git a/api/api.proto b/pkg/api/api.proto similarity index 100% rename from api/api.proto rename to pkg/api/api.proto diff --git a/api/build.sh b/pkg/api/build.sh similarity index 100% rename from api/build.sh rename to pkg/api/build.sh diff --git a/api/python/api_pb2.py b/pkg/api/python/api_pb2.py similarity index 100% rename from api/python/api_pb2.py rename to pkg/api/python/api_pb2.py diff --git a/api/python/api_pb2_grpc.py b/pkg/api/python/api_pb2_grpc.py similarity index 99% rename from api/python/api_pb2_grpc.py rename to pkg/api/python/api_pb2_grpc.py index 65bbea04c79..da810d0cf2a 100644 --- a/api/python/api_pb2_grpc.py +++ b/pkg/api/python/api_pb2_grpc.py @@ -1,7 +1,7 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc -import api.python.api_pb2 as api__pb2 +import pkg.api.python.api_pb2 as api__pb2 class ManagerStub(object): diff --git a/db/db_init.go b/pkg/db/db_init.go similarity index 100% rename from db/db_init.go rename to pkg/db/db_init.go diff --git a/db/interface.go b/pkg/db/interface.go similarity index 99% rename from db/interface.go rename to pkg/db/interface.go index f83eba8b018..346426cb452 100644 --- a/db/interface.go +++ b/pkg/db/interface.go @@ -12,7 +12,7 @@ import ( "strings" "time" - api "github.com/kubeflow/katib/api" + api "github.com/kubeflow/katib/pkg/api" _ "github.com/go-sql-driver/mysql" ) diff --git a/db/interface_test.go b/pkg/db/interface_test.go similarity index 98% rename from db/interface_test.go rename to pkg/db/interface_test.go index 0bb32bcaa0a..93ccf6b5091 100644 --- a/db/interface_test.go +++ b/pkg/db/interface_test.go @@ -14,7 +14,7 @@ import ( "github.com/golang/protobuf/jsonpb" "gopkg.in/DATA-DOG/go-sqlmock.v1" - api "github.com/kubeflow/katib/api" + api "github.com/kubeflow/katib/pkg/api" ) var db_interface VizierDBInterface diff --git a/db/test/test.go b/pkg/db/test/test.go similarity index 85% rename from db/test/test.go rename to pkg/db/test/test.go index 4aa5174f4d1..761d5e2ac49 100644 --- a/db/test/test.go +++ b/pkg/db/test/test.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "github.com/kubeflow/katib/db" + "github.com/kubeflow/katib/pkg/db" "os" ) diff --git a/earlystopping/medianstopping.go b/pkg/earlystopping/medianstopping.go similarity index 98% rename from earlystopping/medianstopping.go rename to pkg/earlystopping/medianstopping.go index 6c07ab32013..408c7f695fd 100644 --- a/earlystopping/medianstopping.go +++ b/pkg/earlystopping/medianstopping.go @@ -3,8 +3,8 @@ package earlystopping import ( "context" "errors" - "github.com/kubeflow/katib/api" - vdb "github.com/kubeflow/katib/db" + "github.com/kubeflow/katib/pkg/api" + vdb "github.com/kubeflow/katib/pkg/db" "log" "sort" "strconv" diff --git a/earlystopping/types.go b/pkg/earlystopping/types.go similarity index 93% rename from earlystopping/types.go rename to pkg/earlystopping/types.go index cd5f6dd560b..d48d8225240 100644 --- a/earlystopping/types.go +++ b/pkg/earlystopping/types.go @@ -3,7 +3,7 @@ package earlystopping import ( "context" - "github.com/kubeflow/katib/api" + "github.com/kubeflow/katib/pkg/api" ) const ( diff --git a/manager/modelstore/modeldb.go b/pkg/manager/modelstore/modeldb.go similarity index 98% rename from manager/modelstore/modeldb.go rename to pkg/manager/modelstore/modeldb.go index fd90688ea42..f24378a33a5 100644 --- a/manager/modelstore/modeldb.go +++ b/pkg/manager/modelstore/modeldb.go @@ -4,8 +4,8 @@ import ( "context" "fmt" "git.apache.org/thrift.git/lib/go/thrift" - "github.com/kubeflow/katib/api" - "github.com/kubeflow/katib/manager/modelstore/modeldb" + "github.com/kubeflow/katib/pkg/api" + "github.com/kubeflow/katib/pkg/manager/modelstore/modeldb" "log" "net" "strconv" diff --git a/manager/modelstore/modeldb/GoUnusedProtection__.go b/pkg/manager/modelstore/modeldb/GoUnusedProtection__.go similarity index 100% rename from manager/modelstore/modeldb/GoUnusedProtection__.go rename to pkg/manager/modelstore/modeldb/GoUnusedProtection__.go diff --git a/manager/modelstore/modeldb/ModelDB-consts.go b/pkg/manager/modelstore/modeldb/ModelDB-consts.go similarity index 100% rename from manager/modelstore/modeldb/ModelDB-consts.go rename to pkg/manager/modelstore/modeldb/ModelDB-consts.go diff --git a/manager/modelstore/modeldb/ModelDB.go b/pkg/manager/modelstore/modeldb/ModelDB.go similarity index 100% rename from manager/modelstore/modeldb/ModelDB.go rename to pkg/manager/modelstore/modeldb/ModelDB.go diff --git a/manager/modelstore/modeldb/model_d_b_service-remote/model_d_b_service-remote.go b/pkg/manager/modelstore/modeldb/model_d_b_service-remote/model_d_b_service-remote.go similarity index 99% rename from manager/modelstore/modeldb/model_d_b_service-remote/model_d_b_service-remote.go rename to pkg/manager/modelstore/modeldb/model_d_b_service-remote/model_d_b_service-remote.go index 85a358106eb..e66fe6ed51f 100755 --- a/manager/modelstore/modeldb/model_d_b_service-remote/model_d_b_service-remote.go +++ b/pkg/manager/modelstore/modeldb/model_d_b_service-remote/model_d_b_service-remote.go @@ -8,7 +8,7 @@ import ( "flag" "fmt" "git.apache.org/thrift.git/lib/go/thrift" - "github.com/kubeflow/katib/manager/modelstore/modeldb" + "github.com/kubeflow/katib/pkg/manager/modelstore/modeldb" "math" "net" "net/url" diff --git a/manager/modelstore/modelstore.go b/pkg/manager/modelstore/modelstore.go similarity index 89% rename from manager/modelstore/modelstore.go rename to pkg/manager/modelstore/modelstore.go index a346ab1d416..96a7b6655f8 100644 --- a/manager/modelstore/modelstore.go +++ b/pkg/manager/modelstore/modelstore.go @@ -1,7 +1,7 @@ package modelstore import ( - "github.com/kubeflow/katib/api" + "github.com/kubeflow/katib/pkg/api" ) type ModelStore interface { diff --git a/manager/visualise/tensorboard/manifest_template/deployment.yaml b/pkg/manager/visualise/tensorboard/manifest_template/deployment.yaml similarity index 100% rename from manager/visualise/tensorboard/manifest_template/deployment.yaml rename to pkg/manager/visualise/tensorboard/manifest_template/deployment.yaml diff --git a/manager/visualise/tensorboard/manifest_template/ingress.yaml b/pkg/manager/visualise/tensorboard/manifest_template/ingress.yaml similarity index 100% rename from manager/visualise/tensorboard/manifest_template/ingress.yaml rename to pkg/manager/visualise/tensorboard/manifest_template/ingress.yaml diff --git a/manager/visualise/tensorboard/manifest_template/service.yaml b/pkg/manager/visualise/tensorboard/manifest_template/service.yaml similarity index 100% rename from manager/visualise/tensorboard/manifest_template/service.yaml rename to pkg/manager/visualise/tensorboard/manifest_template/service.yaml diff --git a/manager/visualise/tensorboard/tensorboardIF.go b/pkg/manager/visualise/tensorboard/tensorboardIF.go similarity index 98% rename from manager/visualise/tensorboard/tensorboardIF.go rename to pkg/manager/visualise/tensorboard/tensorboardIF.go index 7d3d2c04ba7..5954014ee3c 100644 --- a/manager/visualise/tensorboard/tensorboardIF.go +++ b/pkg/manager/visualise/tensorboard/tensorboardIF.go @@ -2,7 +2,7 @@ package tensorboard import ( "bytes" - "github.com/kubeflow/katib/api" + "github.com/kubeflow/katib/pkg/api" "io/ioutil" apiv1 "k8s.io/api/core/v1" exbeatav1 "k8s.io/api/extensions/v1beta1" diff --git a/manager/worker_interface/dlk/dlk.go b/pkg/manager/worker_interface/dlk/dlk.go similarity index 98% rename from manager/worker_interface/dlk/dlk.go rename to pkg/manager/worker_interface/dlk/dlk.go index 8d6b739e3e7..02930cb56cb 100644 --- a/manager/worker_interface/dlk/dlk.go +++ b/pkg/manager/worker_interface/dlk/dlk.go @@ -4,11 +4,11 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/kubeflow/katib/api" - "github.com/kubeflow/katib/db" + "github.com/kubeflow/katib/pkg/api" + "github.com/kubeflow/katib/pkg/db" dlkapi "github.com/kubeflow/katib/dlk/dlkmanager/api" "github.com/kubeflow/katib/dlk/dlkmanager/datastore" - wIF "github.com/kubeflow/katib/manager/worker_interface" + wIF "github.com/kubeflow/katib/pkg/manager/worker_interface" "io/ioutil" "log" "net/http" diff --git a/manager/worker_interface/kubernetes/kubernetes.go b/pkg/manager/worker_interface/kubernetes/kubernetes.go similarity index 97% rename from manager/worker_interface/kubernetes/kubernetes.go rename to pkg/manager/worker_interface/kubernetes/kubernetes.go index 8775bde158b..083c41c0464 100644 --- a/manager/worker_interface/kubernetes/kubernetes.go +++ b/pkg/manager/worker_interface/kubernetes/kubernetes.go @@ -4,20 +4,22 @@ import ( "bytes" "errors" "fmt" - "github.com/kubeflow/katib/api" - "github.com/kubeflow/katib/db" - "github.com/kubeflow/katib/earlystopping" - wIF "github.com/kubeflow/katib/manager/worker_interface" "io/ioutil" + "log" + "strings" + "sync" + "time" + batchv1 "k8s.io/api/batch/v1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8syaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes" - "log" - "strings" - "sync" - "time" + + "github.com/kubeflow/katib/pkg/api" + "github.com/kubeflow/katib/pkg/db" + "github.com/kubeflow/katib/pkg/earlystopping" + wIF "github.com/kubeflow/katib/pkg/manager/worker_interface" ) type KubernetesWorkerInterface struct { @@ -209,6 +211,7 @@ func (d *KubernetesWorkerInterface) CheckRunningTrials(studyId string, objname s } func (d *KubernetesWorkerInterface) SpawnWorkers(trials []*api.Trial, studyId string) error { + // Notice: Missing in the repo. tFile, _ := ioutil.ReadFile("/conf/template.yml") jobs := d.convertTrialToManifest(trials, tFile, studyId) jcl := d.clientset.BatchV1().Jobs(apiv1.NamespaceDefault) diff --git a/manager/worker_interface/nvdocker/nvidia-docker.go b/pkg/manager/worker_interface/nvdocker/nvidia-docker.go similarity index 98% rename from manager/worker_interface/nvdocker/nvidia-docker.go rename to pkg/manager/worker_interface/nvdocker/nvidia-docker.go index a264e0a52bd..d14c139da55 100644 --- a/manager/worker_interface/nvdocker/nvidia-docker.go +++ b/pkg/manager/worker_interface/nvdocker/nvidia-docker.go @@ -8,9 +8,9 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" dclient "github.com/docker/docker/client" - "github.com/kubeflow/katib/api" - "github.com/kubeflow/katib/db" - wIF "github.com/kubeflow/katib/manager/worker_interface" + "github.com/kubeflow/katib/pkg/api" + "github.com/kubeflow/katib/pkg/db" + wIF "github.com/kubeflow/katib/pkg/manager/worker_interface" "io" "io/ioutil" "log" diff --git a/manager/worker_interface/worker_interface.go b/pkg/manager/worker_interface/worker_interface.go similarity index 96% rename from manager/worker_interface/worker_interface.go rename to pkg/manager/worker_interface/worker_interface.go index a0f73d74fa4..82ddf953f42 100644 --- a/manager/worker_interface/worker_interface.go +++ b/pkg/manager/worker_interface/worker_interface.go @@ -3,8 +3,8 @@ package worker_interface import ( "errors" "fmt" - "github.com/kubeflow/katib/api" - "github.com/kubeflow/katib/db" + "github.com/kubeflow/katib/pkg/api" + "github.com/kubeflow/katib/pkg/db" "time" ) diff --git a/mock/api/manager.go b/pkg/mock/api/manager.go similarity index 99% rename from mock/api/manager.go rename to pkg/mock/api/manager.go index 976dac30344..b0629cdd1f6 100644 --- a/mock/api/manager.go +++ b/pkg/mock/api/manager.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/kubeflow/katib/api (interfaces: ManagerClient) +// Source: github.com/kubeflow/katib/pkg/api (interfaces: ManagerClient) // Package mock is a generated GoMock package. package mock @@ -7,7 +7,7 @@ package mock import ( context "context" gomock "github.com/golang/mock/gomock" - api "github.com/kubeflow/katib/api" + api "github.com/kubeflow/katib/pkg/api" grpc "google.golang.org/grpc" reflect "reflect" ) diff --git a/mock/api/suggestion.go b/pkg/mock/api/suggestion.go similarity index 96% rename from mock/api/suggestion.go rename to pkg/mock/api/suggestion.go index 6756f1e2b8a..f4af59e79b5 100644 --- a/mock/api/suggestion.go +++ b/pkg/mock/api/suggestion.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/kubeflow/katib/api (interfaces: SuggestionClient) +// Source: github.com/kubeflow/katib/pkg/api (interfaces: SuggestionClient) // Package mock is a generated GoMock package. package mock @@ -7,7 +7,7 @@ package mock import ( context "context" gomock "github.com/golang/mock/gomock" - api "github.com/kubeflow/katib/api" + api "github.com/kubeflow/katib/pkg/api" grpc "google.golang.org/grpc" reflect "reflect" ) diff --git a/mock/db/db.go b/pkg/mock/db/db.go similarity index 98% rename from mock/db/db.go rename to pkg/mock/db/db.go index 00e19f62edf..e09841e8c7b 100644 --- a/mock/db/db.go +++ b/pkg/mock/db/db.go @@ -1,13 +1,13 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/kubeflow/katib/db (interfaces: VizierDBInterface) +// Source: github.com/kubeflow/katib/pkg/db (interfaces: VizierDBInterface) // Package mock is a generated GoMock package. package mock import ( gomock "github.com/golang/mock/gomock" - api "github.com/kubeflow/katib/api" - db "github.com/kubeflow/katib/db" + api "github.com/kubeflow/katib/pkg/api" + db "github.com/kubeflow/katib/pkg/db" reflect "reflect" time "time" ) diff --git a/mock/modelstore/modelstore.go b/pkg/mock/modelstore/modelstore.go similarity index 96% rename from mock/modelstore/modelstore.go rename to pkg/mock/modelstore/modelstore.go index c5f3e3221a1..90fe89570a1 100644 --- a/mock/modelstore/modelstore.go +++ b/pkg/mock/modelstore/modelstore.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/kubeflow/katib/manager/modelstore (interfaces: ModelStore) +// Source: github.com/kubeflow/katib/pkg/manager/modelstore (interfaces: ModelStore) // Package mock is a generated GoMock package. package mock import ( gomock "github.com/golang/mock/gomock" - api "github.com/kubeflow/katib/api" + api "github.com/kubeflow/katib/pkg/api" reflect "reflect" ) diff --git a/mock/worker/worker.go b/pkg/mock/worker/worker.go similarity index 97% rename from mock/worker/worker.go rename to pkg/mock/worker/worker.go index 9ace51e18ac..916d083e181 100644 --- a/mock/worker/worker.go +++ b/pkg/mock/worker/worker.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/kubeflow/katib/manager/worker_interface (interfaces: WorkerInterface) +// Source: github.com/kubeflow/katib/pkg/manager/worker_interface (interfaces: WorkerInterface) // Package mock is a generated GoMock package. package mock import ( gomock "github.com/golang/mock/gomock" - api "github.com/kubeflow/katib/api" + api "github.com/kubeflow/katib/pkg/api" reflect "reflect" ) diff --git a/suggestion/bayesian_service.py b/pkg/suggestion/bayesian_service.py similarity index 95% rename from suggestion/bayesian_service.py rename to pkg/suggestion/bayesian_service.py index e2b235b75ba..85716d15cd2 100644 --- a/suggestion/bayesian_service.py +++ b/pkg/suggestion/bayesian_service.py @@ -4,10 +4,10 @@ import grpc import numpy as np -from api.python import api_pb2 -from api.python import api_pb2_grpc -from suggestion.bayesianoptimization.src.bayesian_optimization_algorithm import BOAlgorithm -from suggestion.bayesianoptimization.src.algorithm_manager import AlgorithmManager +from pkg.api.python import api_pb2 +from pkg.api.python import api_pb2_grpc +from pkg.suggestion.bayesianoptimization.src.bayesian_optimization_algorithm import BOAlgorithm +from pkg.suggestion.bayesianoptimization.src.algorithm_manager import AlgorithmManager class BayesianService(api_pb2_grpc.SuggestionServicer): diff --git a/suggestion/bayesianoptimization/src/__init__.py b/pkg/suggestion/bayesianoptimization/src/__init__.py similarity index 100% rename from suggestion/bayesianoptimization/src/__init__.py rename to pkg/suggestion/bayesianoptimization/src/__init__.py diff --git a/suggestion/bayesianoptimization/src/acquisition_func/__init__.py b/pkg/suggestion/bayesianoptimization/src/acquisition_func/__init__.py similarity index 100% rename from suggestion/bayesianoptimization/src/acquisition_func/__init__.py rename to pkg/suggestion/bayesianoptimization/src/acquisition_func/__init__.py diff --git a/suggestion/bayesianoptimization/src/acquisition_func/acquisition_func.py b/pkg/suggestion/bayesianoptimization/src/acquisition_func/acquisition_func.py similarity index 95% rename from suggestion/bayesianoptimization/src/acquisition_func/acquisition_func.py rename to pkg/suggestion/bayesianoptimization/src/acquisition_func/acquisition_func.py index d8241b661e7..35d9a0bcae2 100644 --- a/suggestion/bayesianoptimization/src/acquisition_func/acquisition_func.py +++ b/pkg/suggestion/bayesianoptimization/src/acquisition_func/acquisition_func.py @@ -2,7 +2,7 @@ import numpy as np from scipy.stats import norm -from suggestion.bayesianoptimization.src.model.Model import Model +from pkg.suggestion.bayesianoptimization.src.model.Model import Model class AcquisitionFunc: diff --git a/suggestion/bayesianoptimization/src/algorithm_manager.py b/pkg/suggestion/bayesianoptimization/src/algorithm_manager.py similarity index 99% rename from suggestion/bayesianoptimization/src/algorithm_manager.py rename to pkg/suggestion/bayesianoptimization/src/algorithm_manager.py index c64f8475c88..d7d5628403d 100644 --- a/suggestion/bayesianoptimization/src/algorithm_manager.py +++ b/pkg/suggestion/bayesianoptimization/src/algorithm_manager.py @@ -2,7 +2,7 @@ import numpy as np -from api.python import api_pb2 +from pkg.api.python import api_pb2 def deal_with_discrete(feasible_values, current_value): diff --git a/suggestion/bayesianoptimization/src/bayesian_optimization_algorithm.py b/pkg/suggestion/bayesianoptimization/src/bayesian_optimization_algorithm.py similarity index 95% rename from suggestion/bayesianoptimization/src/bayesian_optimization_algorithm.py rename to pkg/suggestion/bayesianoptimization/src/bayesian_optimization_algorithm.py index 987cc46c8dc..62bbc15a71e 100644 --- a/suggestion/bayesianoptimization/src/bayesian_optimization_algorithm.py +++ b/pkg/suggestion/bayesianoptimization/src/bayesian_optimization_algorithm.py @@ -2,7 +2,7 @@ import numpy as np from sklearn.preprocessing import MinMaxScaler -from suggestion.bayesianoptimization.src.global_optimizer.global_optimizer import GlobalOptimizer +from pkg.suggestion.bayesianoptimization.src.global_optimizer.global_optimizer import GlobalOptimizer class BOAlgorithm: diff --git a/suggestion/bayesianoptimization/src/global_optimizer/__init__.py b/pkg/suggestion/bayesianoptimization/src/global_optimizer/__init__.py similarity index 100% rename from suggestion/bayesianoptimization/src/global_optimizer/__init__.py rename to pkg/suggestion/bayesianoptimization/src/global_optimizer/__init__.py diff --git a/suggestion/bayesianoptimization/src/global_optimizer/global_optimizer.py b/pkg/suggestion/bayesianoptimization/src/global_optimizer/global_optimizer.py similarity index 98% rename from suggestion/bayesianoptimization/src/global_optimizer/global_optimizer.py rename to pkg/suggestion/bayesianoptimization/src/global_optimizer/global_optimizer.py index 5aeac887dfe..bcc2f132832 100644 --- a/suggestion/bayesianoptimization/src/global_optimizer/global_optimizer.py +++ b/pkg/suggestion/bayesianoptimization/src/global_optimizer/global_optimizer.py @@ -4,7 +4,7 @@ import copy import numpy as np -from suggestion.bayesianoptimization.src.acquisition_func.acquisition_func import AcquisitionFunc +from pkg.suggestion.bayesianoptimization.src.acquisition_func.acquisition_func import AcquisitionFunc class RectPack: diff --git a/suggestion/bayesianoptimization/src/model/Model.py b/pkg/suggestion/bayesianoptimization/src/model/Model.py similarity index 100% rename from suggestion/bayesianoptimization/src/model/Model.py rename to pkg/suggestion/bayesianoptimization/src/model/Model.py diff --git a/suggestion/bayesianoptimization/src/model/__init__.py b/pkg/suggestion/bayesianoptimization/src/model/__init__.py similarity index 100% rename from suggestion/bayesianoptimization/src/model/__init__.py rename to pkg/suggestion/bayesianoptimization/src/model/__init__.py diff --git a/suggestion/grid_service.go b/pkg/suggestion/grid_service.go similarity index 99% rename from suggestion/grid_service.go rename to pkg/suggestion/grid_service.go index ff8bed6730d..73a1b30bc8c 100644 --- a/suggestion/grid_service.go +++ b/pkg/suggestion/grid_service.go @@ -6,7 +6,7 @@ import ( "log" "strconv" - "github.com/kubeflow/katib/api" + "github.com/kubeflow/katib/pkg/api" ) type GridSuggestParameters struct { diff --git a/suggestion/hyperband_service.go b/pkg/suggestion/hyperband_service.go similarity index 99% rename from suggestion/hyperband_service.go rename to pkg/suggestion/hyperband_service.go index 607464683a2..fa90c86bded 100644 --- a/suggestion/hyperband_service.go +++ b/pkg/suggestion/hyperband_service.go @@ -4,13 +4,13 @@ import ( "context" "crypto/rand" "fmt" - "github.com/kubeflow/katib/db" + "github.com/kubeflow/katib/pkg/db" "log" "math" "sort" "strconv" - "github.com/kubeflow/katib/api" + "github.com/kubeflow/katib/pkg/api" ) type Bracket []*api.Trial diff --git a/suggestion/random_service.go b/pkg/suggestion/random_service.go similarity index 99% rename from suggestion/random_service.go rename to pkg/suggestion/random_service.go index c180d13c42e..e4096da5022 100644 --- a/suggestion/random_service.go +++ b/pkg/suggestion/random_service.go @@ -7,7 +7,7 @@ import ( "strconv" "time" - "github.com/kubeflow/katib/api" + "github.com/kubeflow/katib/pkg/api" ) type RandomSuggestParameters struct { diff --git a/suggestion/test_client.py b/pkg/suggestion/test_client.py similarity index 96% rename from suggestion/test_client.py rename to pkg/suggestion/test_client.py index 7dda4362c14..5543f0353aa 100644 --- a/suggestion/test_client.py +++ b/pkg/suggestion/test_client.py @@ -1,9 +1,9 @@ import grpc -from api.python import api_pb2 -from api.python import api_pb2_grpc -from suggestion.test_func import func -from suggestion.types import DEFAULT_PORT +from pkg.api.python import api_pb2 +from pkg.api.python import api_pb2_grpc +from pkg.suggestion.test_func import func +from pkg.suggestion.types import DEFAULT_PORT def run(): diff --git a/suggestion/test_func.py b/pkg/suggestion/test_func.py similarity index 100% rename from suggestion/test_func.py rename to pkg/suggestion/test_func.py diff --git a/suggestion/types.go b/pkg/suggestion/types.go similarity index 92% rename from suggestion/types.go rename to pkg/suggestion/types.go index 0a42ca89d30..3187e92800d 100644 --- a/suggestion/types.go +++ b/pkg/suggestion/types.go @@ -3,7 +3,7 @@ package suggestion import ( "context" - "github.com/kubeflow/katib/api" + "github.com/kubeflow/katib/pkg/api" ) const ( diff --git a/suggestion/types.py b/pkg/suggestion/types.py similarity index 100% rename from suggestion/types.py rename to pkg/suggestion/types.py diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 00000000000..888cbceb18a --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright 2018 The Kubeflow Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +PREFIX="katib" +CMD_PREFIX="cmd" + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +cd ${SCRIPT_ROOT} + +echo "Building core image..." +docker build -t ${PREFIX}/vizier-core -f ${CMD_PREFIX}/manager/Dockerfile . + +echo "Building suggestion images..." +docker build -t ${PREFIX}/suggestion-random -f ${CMD_PREFIX}/suggestion/random/Dockerfile . +docker build -t ${PREFIX}/suggestion-grid -f ${CMD_PREFIX}/suggestion/grid/Dockerfile . +docker build -t ${PREFIX}/suggestion-hyperband -f ${CMD_PREFIX}/suggestion/hyperband/Dockerfile . +docker build -t ${PREFIX}/suggestion-bayesianoptimization -f ${CMD_PREFIX}/suggestion/bayesianoptimization/Dockerfile . + +echo "Building earlystopping images..." +docker build -t ${PREFIX}/earlystopping-medianstopping -f ${CMD_PREFIX}/earlystopping/medianstopping/Dockerfile . + +echo "Building dlk manager image..." +docker build -t ${PREFIX}/dlk-manager -f dlk/Dockerfile . + +echo "Building UI image..." +docker build -t ${PREFIX}/katib-frontend -f modeldb/Dockerfile . + +echo "Building CLI image..." +docker build -t ${PREFIX}/katib-cli -f ${CMD_PREFIX}/cli/Dockerfile . +cd - > /dev/null diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 00000000000..3afeb8e42f3 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2018 The Kubeflow Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +cd ${SCRIPT_ROOT} +kubectl apply -f manifests/0-namespace.yaml +kubectl apply -f manifests/modeldb/db +kubectl apply -f manifests/modeldb/backend +kubectl apply -f manifests/modeldb/frontend +kubectl apply -f manifests/dlk +kubectl apply -f manifests/vizier/db +kubectl apply -f manifests/vizier/core +kubectl apply -f manifests/vizier/suggestion/random +cd - > /dev/null diff --git a/scripts/mockgen.sh b/scripts/mockgen.sh index 697d72ad6a0..e13cabbc40f 100755 --- a/scripts/mockgen.sh +++ b/scripts/mockgen.sh @@ -22,13 +22,13 @@ SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. cd ${SCRIPT_ROOT} echo "Generating ManagerClient..." -mockgen -package mock -destination mock/api/manager.go github.com/kubeflow/katib/api ManagerClient +mockgen -package mock -destination pkg/mock/api/manager.go github.com/kubeflow/katib/pkg/api ManagerClient echo "Generating SuggestionClient..." -mockgen -package mock -destination mock/api/suggestion.go github.com/kubeflow/katib/api SuggestionClient +mockgen -package mock -destination pkg/mock/api/suggestion.go github.com/kubeflow/katib/pkg/api SuggestionClient echo "Generating VizierDBInterface..." -mockgen -package mock -destination mock/db/db.go github.com/kubeflow/katib/db VizierDBInterface +mockgen -package mock -destination pkg/mock/db/db.go github.com/kubeflow/katib/pkg/db VizierDBInterface echo "Generating WorkerInterface..." -mockgen -package mock -destination mock/worker/worker.go github.com/kubeflow/katib/manager/worker_interface WorkerInterface +mockgen -package mock -destination pkg/mock/worker/worker.go github.com/kubeflow/katib/pkg/manager/worker_interface WorkerInterface echo "Generating ModelStore..." -mockgen -package mock -destination mock/modelstore/modelstore.go github.com/kubeflow/katib/manager/modelstore ModelStore +mockgen -package mock -destination pkg/mock/modelstore/modelstore.go github.com/kubeflow/katib/pkg/manager/modelstore ModelStore cd - > /dev/null diff --git a/test/scripts/build.sh b/test/scripts/build.sh index 5189c9b0d88..3534ead5941 100755 --- a/test/scripts/build.sh +++ b/test/scripts/build.sh @@ -36,29 +36,29 @@ ln -s ${PWD} ${GO_DIR} cd ${GO_DIR} echo "Build operator binary" mkdir bin -go build -o bin/katib-core github.com/kubeflow/katib/manager -go build -o bin/dlkmanager github.com/kubeflow/katib//dlk/dlkmanager -go build -o bin/katib-suggestion-grid github.com/kubeflow/katib/suggestion/grid -go build -o bin/katib-suggestion-hyperband github.com/kubeflow/katib/suggestion/hyperband -go build -o bin/katib-suggestion-random github.com/kubeflow/katib/suggestion/random -go build -o bin/katib-earlystopping-medianstopping github.com/kubeflow/katib/earlystopping/medianstopping -go build -o bin/katib github.com/kubeflow/katib/cli +go build -o bin/katib-core github.com/kubeflow/katib/cmd/manager +go build -o bin/dlkmanager github.com/kubeflow/katib/dlk/dlkmanager +go build -o bin/katib-suggestion-grid github.com/kubeflow/katib/cmd/suggestion/grid +go build -o bin/katib-suggestion-hyperband github.com/kubeflow/katib/cmd/suggestion/hyperband +go build -o bin/katib-suggestion-random github.com/kubeflow/katib/cmd/suggestion/random +go build -o bin/katib-earlystopping-medianstopping github.com/kubeflow/katib/cmd/earlystopping/medianstopping +go build -o bin/katib github.com/kubeflow/katib/cmd/cli #echo "building container in gcloud" #gcloud version # gcloud components update -q -cp manager/Dockerfile . +cp cmd/manager/Dockerfile . gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/vizier-core:${VERSION} --project=${PROJECT} -cp suggestion/random/Dockerfile . +cp cmd/suggestion/random/Dockerfile . gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-random:${VERSION} --project=${PROJECT} -cp suggestion/grid/Dockerfile . +cp cmd/suggestion/grid/Dockerfile . gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-grid:${VERSION} --project=${PROJECT} -cp suggestion/hyperband/Dockerfile . +cp cmd/suggestion/hyperband/Dockerfile . gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-hyperband:${VERSION} --project=${PROJECT} -cp suggestion/bayesianoptimization/Dockerfile . +cp cmd/suggestion/bayesianoptimization/Dockerfile . gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-bayesianoptimization:${VERSION} --project=${PROJECT} -cp earlystopping/medianstopping/Dockerfile . +cp cmd/earlystopping/medianstopping/Dockerfile . gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/earlystopping-medianstopping:${VERSION} --project=${PROJECT} cp dlk/Dockerfile . gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/dlk-manager:${VERSION} --project=${PROJECT} -cp modeldb//Dockerfile . +cp modeldb/Dockerfile . gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/katib-frontend:${VERSION} --project=${PROJECT} diff --git a/test/scripts/run-tests.sh b/test/scripts/run-tests.sh index 8fc4a089b69..6f1f8ef7ac4 100755 --- a/test/scripts/run-tests.sh +++ b/test/scripts/run-tests.sh @@ -51,7 +51,7 @@ sed -i -e "s@image: katib\/earlystopping-medianstopping@image: ${REGISTRY}\/${RE sed -i -e "s@image: katib\/dlk-manager@image: ${REGISTRY}\/${REPO_NAME}\/dlk-manager:${VERSION}@" manifests/dlk/deployment.yaml sed -i -e "s@image: katib\/katib-frontend@image: ${REGISTRY}\/${REPO_NAME}\/katib-frontend:${VERSION}@" manifests/modeldb/frontend/deployment.yaml cd ${GO_DIR} -./deploy.sh +./scripts/deploy.sh TIMEOUT=60 PODNUM=$(kubectl get pods -n katib | grep -v NAME | wc -l)