Migrate register from docker to flytectl (flyteorg#257)

* Migrate register from docker to flytectl Signed-off-by: yuvraj <[email protected]> * Small fix Signed-off-by: Yuvraj <[email protected]> * added docs for flytectl in readme Signed-off-by: Yuvraj <[email protected]> * remove register command from conatiner Signed-off-by: Yuvraj <[email protected]> * use register command outside the container Signed-off-by: Yuvraj <[email protected]> * remove register from container makefile for all example Signed-off-by: Yuvraj <[email protected]> * fix typo Signed-off-by: Yuvraj <[email protected]> * revert conatiner change Signed-off-by: Yuvraj <[email protected]>
eapolinario · Jun 28, 2021 · 1d2f279 · 1d2f279
1 parent fc5887b
commit 1d2f279
Show file tree

Hide file tree

Showing 12 changed files with 33 additions and 54 deletions.
diff --git a/.github/workflows/release_examples.yml b/.github/workflows/release_examples.yml
@@ -37,7 +37,7 @@ jobs:
         run: |
           mkdir -p cookbook/release-snacks /tmp/release-snacks-${{ matrix.directory }}/
           make -C cookbook archive
-          cp -r /tmp/release-snacks-${{ matrix.directory }}/* cookbook/release-snacks/
+          cp -r cookbook/release-snacks/* /tmp/release-snacks-${{ matrix.directory }}/
   releaser:
     name: Release example
     runs-on: ubuntu-latest

diff --git a/.github/workflows/sandbox_register.yml b/.github/workflows/sandbox_register.yml
@@ -22,7 +22,11 @@ jobs:
         with:
           fetch-depth: "0"
       - uses: unionai/[email protected]
-      - name: setup sandbox
-        run: flytectl sandbox start --sourcesPath=$GITHUB_WORKSPACE
-      - name: fast register
+      - name: Setup sandbox
+        run: make setup
+      - name: Setup flytectl config
+        run: eval $(make flytectl-config)
+      - name: Register
+        run: VERSION=latest make register
+      - name: Fast register
         run: REGISTRY=ghcr.io/flyteorg VERSION=latest make fast_register
diff --git a/.gitignore b/.gitignore
@@ -22,4 +22,6 @@ _build/
 .vscode/
 *-out.html
 *-out.ipynb
-.python-version
+.python-version
+./cookbook/release-snacks/*.tgz
+./cookbook/flyte_tests_manifest.json
diff --git a/Makefile b/Makefile
@@ -48,7 +48,7 @@ endif
 .PHONY: setup
 setup:
 	$(call LOG,Starting Flyte sandbox)
-	flytectl sandbox start --sourcesPath:=$(shell pwd)
+	flytectl sandbox start --sourcesPath=$(shell pwd)
 
 .PHONY: start
 start: setup
@@ -64,6 +64,10 @@ teardown: _requires-sandbox-up  ## Teardown Flyte sandbox
 
 .PHONY: status
 status: _requires-sandbox-up  ## Show status of Flyte deployment
+	$(call LOG,Flytectl sandbox status \n)
+	flytectl sandbox status
+	$(call LOG,\n)
+	$(call LOG,Flyte kubernetes deployment status \n)
 	kubectl get pods -n flyte
 
 .PHONY: shell
@@ -73,7 +77,8 @@ shell: _requires-sandbox-up  ## Drop into a development shell
 .PHONY: register
 register: _requires-sandbox-up  ## Register Flyte cookbook workflows
 	$(call LOG,Registering example workflows in cookbook/$(EXAMPLES_MODULE))
-	$(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) register)
+	make -C cookbook/$(EXAMPLES_MODULE) register
+
 
 .PHONY: fast_register
 fast_register: _requires-sandbox-up  ## Fast register Flyte cookbook workflows
@@ -86,4 +91,9 @@ kubectl-config:
 	# Makefiles run recipes in sub-processes. A sub-process cannot modify the parent process's environment.
 	# The best I (@EngHabu) can think of at the moment is to output this for the user to eval in the
 	# parent process.
-	echo "export KUBECONFIG=$(KUBECONFIG):~/.kube/config:$(KUBE_CONFIG)/k3s/k3s.yaml"
+	echo "export KUBECONFIG=$(KUBECONFIG):~/.kube/config:$(FLYTE_DIR)/k3s/k3s.yaml"
+
+.PHONY: flytectl-config
+flytectl-config:
+	mkdir -p ~/.flyte
+	cp $(FLYTE_DIR)/config-sandbox.yaml $(FLYTE_DIR)/config.yaml
diff --git a/README.md b/README.md
@@ -78,6 +78,7 @@ Here are the setup instructions to start contributing to `flytesnacks` repo:
 4. Test your code!
     * If the Python code can be run locally, just use `python <my file>` to run it.
     * If the Python code has to be tested in a cluster:
+        * Install [flytectl](https://docs.flyte.org/projects/flytectl/en/stable/) by running `brew install flyteorg/homebrew-tap/flytectl`. 
         * Run the `make start` command in the root directory of the flytesnacks repo
         * Visit https://localhost:30081 to view the Flyte console consisting of the examples present in flytesnacks/cookbook/core directory
         * To fetch new dependencies and rebuild the image, run `make register`

diff --git a/cookbook/Makefile b/cookbook/Makefile
@@ -32,7 +32,3 @@ run-examples: ## Runs all examples
 .PHONY: archive
 archive: # archive directories (e.g. _pb_output/)
 	bash scripts/ci_helper.sh RELEASE
-
-.PHONY: register-workflow-ci
-register-workflow-ci:
-	bash scripts/ci_helper.sh REGISTER
diff --git a/cookbook/case_studies/in_container.mk b/cookbook/case_studies/in_container.mk
@@ -11,10 +11,6 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register: serialize
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)

diff --git a/cookbook/common/leaf.mk b/cookbook/common/leaf.mk
@@ -3,6 +3,7 @@
 # This is used by the image building script referenced below. Normally it just takes the directory name but in this
 # case we want it to be called something else.
 IMAGE_NAME=flytecookbook
+FLYTE_CONFIG := ~/.flyte/config-sandbox.yaml
 export VERSION ?= $(shell git rev-parse HEAD)
 
 define PIP_COMPILE
@@ -45,6 +46,10 @@ endif
 # The Flyte project that we want to register under
 export PROJECT ?= flytesnacks
 
+
+# The Flyte domain that we want to register under
+export DOMAIN ?= development
+
 # If the REGISTRY environment variable has been set, that means the image name will not just be tagged as
 #   flytecookbook:<sha> but rather,
 #   ghcr.io/flyteorg/flytecookbook:<sha> or whatever your REGISTRY is.
@@ -138,25 +143,10 @@ serialize: clean _pb_output docker_build
 
 
 .PHONY: register
-register: clean _pb_output docker_push
+register: clean _pb_output serialize docker_push
 	@echo ${VERSION}
 	@echo ${CURDIR}
-	docker run -i --rm \
-		--network host \
-		-e REGISTRY=${REGISTRY} \
-		-e MAKEFLAGS=${MAKEFLAGS} \
-		-e FLYTE_HOST=${FLYTE_HOST} \
-		-e INSECURE_FLAG=${INSECURE_FLAG} \
-		-e PROJECT=${PROJECT} \
-		-e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \
-		-e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \
-		-e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \
-		-e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \
-		-e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \
-		-e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \
-		-e VERSION=${VERSION} \
-		-v ${CURDIR}/_pb_output:/tmp/output \
-		${TAGGED_IMAGE} make register
+	flytectl register file ${CURDIR}/_pb_output/* -d ${DOMAIN} -p ${PROJECT}  --config ${FLYTE_CONFIG} --version=${VERSION}
 
 _pb_output:
 	mkdir -p _pb_output

diff --git a/cookbook/core/in_container.mk b/cookbook/core/in_container.mk
@@ -11,10 +11,6 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register: serialize
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)

diff --git a/cookbook/in_container.mk b/cookbook/in_container.mk
@@ -11,14 +11,10 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register: serialize
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
 .PHONY: fast_register
 fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
+	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/integrations/in_container.mk b/cookbook/integrations/in_container.mk
@@ -11,10 +11,6 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register: serialize
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)

diff --git a/cookbook/scripts/ci_helper.sh b/cookbook/scripts/ci_helper.sh
@@ -10,14 +10,6 @@ release_example() {
   done
 }
 
-register_example() {
-  for row in $(cat flyte_tests_manifest.json | jq -c '.[]'); do
-    flytectl register file "./release-snacks/flytesnacks-$(echo ${row} | jq -r '.name').tgz" -p flytesnacks -d development --archive
-  done
-}
-
 if [ "$1" == "RELEASE" ]; then
   release_example
-elif [ $1 == "REGISTER" ]; then
-  register_example
 fi