migrate flyte-cli with flytectl (flyteorg#409)

* migrate flyte-cli with flytectl Signed-off-by: Yuvraj <[email protected]>
eapolinario · Sep 28, 2021 · f4bb0a0 · f4bb0a0
1 parent 466f775
commit f4bb0a0
Show file tree

Hide file tree

Showing 12 changed files with 33 additions and 117 deletions.
diff --git a/Makefile b/Makefile
@@ -50,6 +50,7 @@ endif
 setup:
 	$(call LOG,Starting Flyte sandbox)
 	flytectl sandbox start --source=$(shell pwd)
+	flytectl config init
 
 .PHONY: start
 start: setup
@@ -74,12 +75,14 @@ shell: _requires-sandbox-up  ## Drop into a development shell
 .PHONY: register
 register: _requires-sandbox-up  ## Register Flyte cookbook workflows
 	$(call LOG,Registering example workflows in cookbook/$(EXAMPLES_MODULE))
-	$(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) register)
+	$(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) serialize)
+	make -C cookbook/$(EXAMPLES_MODULE) register
 
 .PHONY: fast_register
 fast_register: _requires-sandbox-up  ## Fast register Flyte cookbook workflows
-	$(call LOG,Fast registering example workflows in cookbook/$(EXAMPLES_MODULE))
-	$(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) fast_register)
+	$(call LOG,Fast registering example workflows from latest release of flytesnacks)
+	$(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) fast_serialize)
+	make -C cookbook/$(EXAMPLES_MODULE) fast_register
 
 .PHONY: setup-kubectl
 kubectl-config:

diff --git a/README.md b/README.md
@@ -75,8 +75,8 @@ Here are the setup instructions to start contributing to `flytesnacks` repo:
         * Install flytectl by running `brew install flyteorg/homebrew-tap/flytectl`. Learn more about install and configuration of [flytectl](https://docs.flyte.org/projects/flytectl/en/latest/index.html) 
         * Run the `make start` command in the root directory of the flytesnacks repo
         * Visit https://localhost:30081/console to view the Flyte console consisting of the examples present in flytesnacks/cookbook/core directory
-        * To fetch new dependencies and rebuild the image, run `make register`
-        * If examples from a different directory (other than `core`) have to registered, enter the sandbox first: `make shell` and then run `make -C <directory> register`.
+        * To fetch new dependencies and rebuild the image, run `make serialize`
+        * If examples from a different directory (other than `core`) have to registered then run `make -C <directory> register`.
 
 ### 📝 Contribute to documentation
 

diff --git a/cookbook/case_studies/feature_engineering/in_container.mk b/cookbook/case_studies/feature_engineering/in_container.mk
@@ -11,14 +11,6 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register:
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
-
-.PHONY: fast_register
-fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/case_studies/in_container.mk b/cookbook/case_studies/in_container.mk
@@ -11,14 +11,7 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register:
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
 
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
-
-.PHONY: fast_register
-fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/case_studies/ml_training/in_container.mk b/cookbook/case_studies/ml_training/in_container.mk
@@ -11,14 +11,6 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register:
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
-
-.PHONY: fast_register
-fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/common/leaf.mk b/cookbook/common/leaf.mk
@@ -24,12 +24,6 @@ export FLYTE_AWS_ACCESS_KEY_ID ?= minio
 # Used to authenticate to s3. For a production AWS S3, it's discouraged to use keys and key ids.
 export FLYTE_AWS_SECRET_ACCESS_KEY ?= miniostorage
 
-# Used to publish artifacts for fast registration
-export ADDL_DISTRIBUTION_DIR ?= s3://my-s3-bucket/fast/
-
-# The base of where Blobs, Schemas and other offloaded types are, by default, serialized.
-export OUTPUT_DATA_PREFIX ?= s3://my-s3-bucket/raw-data
-
 # Instructs flyte-cli commands to use insecure channel when communicating with Flyte's control plane.
 # If you're port-forwarding your service or running the sandbox Flyte deployment, specify INSECURE=1 before your make command.
 # If your Flyte Admin is behind SSL, don't specify anything.
@@ -45,6 +39,8 @@ endif
 # The Flyte project that we want to register under
 export PROJECT ?= flytesnacks
 
+export DOMAIN ?= development
+
 # If the REGISTRY environment variable has been set, that means the image name will not just be tagged as
 #   flytecookbook:<sha> but rather,
 #   ghcr.io/flyteorg/flytecookbook:<sha> or whatever your REGISTRY is.
@@ -60,6 +56,14 @@ ifndef SERVICE_ACCOUNT
 	SERVICE_ACCOUNT=default
 endif
 
+ifndef ADDL_DISTRIBUTION_DIR
+	ADDL_DISTRIBUTION_DIR=s3://my-s3-bucket/fast/
+endif
+
+ifndef OUTPUT_DATA_PREFIX
+	OUTPUT_DATA_PREFIX=s3://my-s3-bucket/raw-data
+endif
+
 requirements.txt: export CUSTOM_COMPILE_COMMAND := $(MAKE) requirements.txt
 requirements.txt: requirements.in install-piptools
 	$(call PIP_COMPILE,requirements.in)
@@ -89,28 +93,17 @@ fast_serialize: clean _pb_output
 		${TAGGED_IMAGE} make fast_serialize
 
 .PHONY: fast_register
-fast_register: fast_serialize ## Packages code and registers without building docker images.
+fast_register: ## Packages code and registers without building docker images.
 	@echo "Tagged Image: "
 	@echo ${TAGGED_IMAGE}
 	@echo ${CURDIR}
-	docker run -it --rm \
-		--network host \
-		-e SANDBOX=${SANDBOX} \
-		-e REGISTRY=${REGISTRY} \
-		-e MAKEFLAGS=${MAKEFLAGS} \
-		-e FLYTE_HOST=${FLYTE_HOST} \
-		-e INSECURE_FLAG=${INSECURE_FLAG} \
-		-e PROJECT=${PROJECT} \
-		-e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \
-		-e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \
-		-e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \
-		-e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \
-		-e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \
-		-e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \
-		-e VERSION=${VERSION} \
-		-v ${CURDIR}/_pb_output:/tmp/output \
-		-v ${CURDIR}:/root/$(shell basename $(CURDIR)) \
-		${TAGGED_IMAGE} make fast_register
+	flytectl register files ${CURDIR}/_pb_output/* \
+		-p ${PROJECT} \
+		-d ${DOMAIN} \
+		--outputLocationPrefix ${OUTPUT_DATA_PREFIX} \
+		--k8sServiceAccount $(SERVICE_ACCOUNT) \
+		--version fast${VERSION} \
+		--sourceUploadPath ${ADDL_DISTRIBUTION_DIR}
 
 .PHONY: docker_build
 docker_build:
@@ -141,26 +134,15 @@ serialize: clean _pb_output docker_build
 
 
 .PHONY: register
-register: serialize docker_push
+register: docker_push
 	@echo ${VERSION}
 	@echo ${CURDIR}
-	docker run -i --rm \
-		--network host \
-		-e SANDBOX=${SANDBOX} \
-		-e REGISTRY=${REGISTRY} \
-		-e MAKEFLAGS=${MAKEFLAGS} \
-		-e FLYTE_HOST=${FLYTE_HOST} \
-		-e INSECURE_FLAG=${INSECURE_FLAG} \
-		-e PROJECT=${PROJECT} \
-		-e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \
-		-e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \
-		-e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \
-		-e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \
-		-e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \
-		-e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \
-		-e VERSION=${VERSION} \
-		-v ${CURDIR}/_pb_output:/tmp/output \
-		${TAGGED_IMAGE} make register
+	flytectl register files ${CURDIR}/_pb_output/* \
+		-p ${PROJECT} \
+		-d ${DOMAIN} \
+		--outputLocationPrefix ${OUTPUT_DATA_PREFIX} \
+		--k8sServiceAccount $(SERVICE_ACCOUNT) \
+		--version ${VERSION}
 
 _pb_output:
 	mkdir -p _pb_output

diff --git a/cookbook/core/in_container.mk b/cookbook/core/in_container.mk
@@ -11,14 +11,6 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register:
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
-
-.PHONY: fast_register
-fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/in_container.mk b/cookbook/in_container.mk
@@ -11,14 +11,7 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register:
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
 
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
-
-.PHONY: fast_register
-fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/integrations/aws/in_container.mk b/cookbook/integrations/aws/in_container.mk
@@ -11,14 +11,7 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register:
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
 
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
-
-.PHONY: fast_register
-fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/integrations/flytekit_plugins/in_container.mk b/cookbook/integrations/flytekit_plugins/in_container.mk
@@ -11,14 +11,6 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register:
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
-
-.PHONY: fast_register
-fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/integrations/in_container.mk b/cookbook/integrations/in_container.mk
@@ -11,14 +11,6 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register:
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
-
-.PHONY: fast_register
-fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/integrations/kubernetes/in_container.mk b/cookbook/integrations/kubernetes/in_container.mk
@@ -11,14 +11,6 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
-.PHONY: register
-register:
-	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
-
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
-
-.PHONY: fast_register
-fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*