Revert "Migrate register from docker to flytectl" (flyteorg#315)

* Revert "Migrate register from docker to flytectl (flyteorg#257)"

This reverts commit 1d2f279.

.github/workflows/release_examples.yml

@@ -1,10 +1,6 @@
 name: Serialize & Release Examples
 
 on:
-  pull_request:
-    paths-ignore:
-      - 'cookbook/docs/**'
-      - 'cookbook/deployment/**'
   create:
     tags:
       - v*
@@ -21,7 +17,6 @@ jobs:
         with:
           fetch-depth: "0"
       - uses: actions/cache@v1
-        if: github.event_name != 'pull_request'
         with:
           path: /tmp/release-snacks-${{ matrix.directory }}/
           key: release-snacks-${{ matrix.directory }}-${{ github.sha }}
@@ -31,17 +26,15 @@ jobs:
         run: |
           REGISTRY=ghcr.io/flyteorg VERSION=latest make -C cookbook/${{ matrix.directory }} serialize
       - name: Cache Serialized protos
-        if: github.event_name != 'pull_request'
         env:
           REGISTRY: "ghcr.io/flyteorg"
         run: |
           mkdir -p cookbook/release-snacks /tmp/release-snacks-${{ matrix.directory }}/
           make -C cookbook archive
-          cp -r cookbook/release-snacks/* /tmp/release-snacks-${{ matrix.directory }}/
+          cp -r cookbook/release-snacks/ /tmp/release-snacks-${{ matrix.directory }}/*
   releaser:
     name: Release example
     runs-on: ubuntu-latest
-    if: github.event_name != 'pull_request'
     needs: [ serialize-example ]
     steps:
       - uses: actions/checkout@v2

.github/workflows/sandbox_register.yml

@@ -22,11 +22,7 @@ jobs:
         with:
           fetch-depth: "0"
       - uses: unionai/[email protected]
-      - name: Setup sandbox
-        run: make setup
-      - name: Setup flytectl config
-        run: eval $(make flytectl-config)
-      - name: Register
-        run: VERSION=latest make register
-      - name: Fast register
+      - name: setup sandbox
+        run: flytectl sandbox start --sourcesPath=$GITHUB_WORKSPACE
+      - name: fast register
         run: REGISTRY=ghcr.io/flyteorg VERSION=latest make fast_register

.gitignore

@@ -22,6 +22,4 @@ _build/
 .vscode/
 *-out.html
 *-out.ipynb
-.python-version
-./cookbook/release-snacks/*.tgz
-./cookbook/flyte_tests_manifest.json
+.python-version

Makefile

@@ -10,6 +10,7 @@ MINIO_PROXY_PORT := 30084
 FLYTE_SANDBOX_NAME := flyte-sandbox
 FLYTE_DIR := ~/.flyte
 
+
 # Module of cookbook examples to register
 EXAMPLES_MODULE := core
 
@@ -64,10 +65,6 @@ teardown: _requires-sandbox-up  ## Teardown Flyte sandbox
 
 .PHONY: status
 status: _requires-sandbox-up  ## Show status of Flyte deployment
-	$(call LOG,Flytectl sandbox status \n)
-	flytectl sandbox status
-	$(call LOG,\n)
-	$(call LOG,Flyte kubernetes deployment status \n)
 	kubectl get pods -n flyte
 
 .PHONY: shell
@@ -77,8 +74,7 @@ shell: _requires-sandbox-up  ## Drop into a development shell
 .PHONY: register
 register: _requires-sandbox-up  ## Register Flyte cookbook workflows
 	$(call LOG,Registering example workflows in cookbook/$(EXAMPLES_MODULE))
-	make -C cookbook/$(EXAMPLES_MODULE) register
-
+	$(call RUN_IN_SANDBOX,make -C cookbook/$(EXAMPLES_MODULE) register)
 
 .PHONY: fast_register
 fast_register: _requires-sandbox-up  ## Fast register Flyte cookbook workflows
@@ -92,8 +88,3 @@ kubectl-config:
 	# The best I (@EngHabu) can think of at the moment is to output this for the user to eval in the
 	# parent process.
 	echo "export KUBECONFIG=$(KUBECONFIG):~/.kube/config:$(FLYTE_DIR)/k3s/k3s.yaml"
-
-.PHONY: flytectl-config
-flytectl-config:
-	mkdir -p ~/.flyte
-	cp $(FLYTE_DIR)/config-sandbox.yaml $(FLYTE_DIR)/config.yaml

README.md

@@ -78,7 +78,6 @@ Here are the setup instructions to start contributing to `flytesnacks` repo:
 4. Test your code!
     * If the Python code can be run locally, just use `python <my file>` to run it.
     * If the Python code has to be tested in a cluster:
-        * Install [flytectl](https://docs.flyte.org/projects/flytectl/en/stable/) by running `brew install flyteorg/homebrew-tap/flytectl`. 
         * Run the `make start` command in the root directory of the flytesnacks repo
         * Visit https://localhost:30081 to view the Flyte console consisting of the examples present in flytesnacks/cookbook/core directory
         * To fetch new dependencies and rebuild the image, run `make register`

cookbook/Makefile

@@ -32,3 +32,7 @@ run-examples: ## Runs all examples
 .PHONY: archive
 archive: # archive directories (e.g. _pb_output/)
 	bash scripts/ci_helper.sh RELEASE
+
+.PHONY: register-workflow-ci
+register-workflow-ci:
+	bash scripts/ci_helper.sh REGISTER

cookbook/case_studies/in_container.mk

@@ -11,6 +11,10 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
+.PHONY: register
+register: serialize
+	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
+
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)

cookbook/common/leaf.mk

@@ -3,7 +3,6 @@
 # This is used by the image building script referenced below. Normally it just takes the directory name but in this
 # case we want it to be called something else.
 IMAGE_NAME=flytecookbook
-FLYTE_CONFIG := ~/.flyte/config-sandbox.yaml
 export VERSION ?= $(shell git rev-parse HEAD)
 
 define PIP_COMPILE
@@ -46,10 +45,6 @@ endif
 # The Flyte project that we want to register under
 export PROJECT ?= flytesnacks
 
-
-# The Flyte domain that we want to register under
-export DOMAIN ?= development
-
 # If the REGISTRY environment variable has been set, that means the image name will not just be tagged as
 #   flytecookbook:<sha> but rather,
 #   ghcr.io/flyteorg/flytecookbook:<sha> or whatever your REGISTRY is.
@@ -143,10 +138,25 @@ serialize: clean _pb_output docker_build
 
 
 .PHONY: register
-register: clean _pb_output serialize docker_push
+register: clean _pb_output docker_push
 	@echo ${VERSION}
 	@echo ${CURDIR}
-	flytectl register file ${CURDIR}/_pb_output/* -d ${DOMAIN} -p ${PROJECT}  --config ${FLYTE_CONFIG} --version=${VERSION}
+	docker run -i --rm \
+		--network host \
+		-e REGISTRY=${REGISTRY} \
+		-e MAKEFLAGS=${MAKEFLAGS} \
+		-e FLYTE_HOST=${FLYTE_HOST} \
+		-e INSECURE_FLAG=${INSECURE_FLAG} \
+		-e PROJECT=${PROJECT} \
+		-e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \
+		-e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \
+		-e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \
+		-e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \
+		-e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \
+		-e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \
+		-e VERSION=${VERSION} \
+		-v ${CURDIR}/_pb_output:/tmp/output \
+		${TAGGED_IMAGE} make register
 
 _pb_output:
 	mkdir -p _pb_output

cookbook/core/in_container.mk

@@ -11,6 +11,10 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
+.PHONY: register
+register: serialize
+	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
+
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)

cookbook/in_container.mk

@@ -11,10 +11,14 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
+.PHONY: register
+register: serialize
+	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
+
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
 .PHONY: fast_register
 fast_register: fast_serialize
-	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
+	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*

cookbook/integrations/in_container.mk

@@ -11,6 +11,10 @@ $(SERIALIZED_PB_OUTPUT_DIR): clean
 serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
 
+.PHONY: register
+register: serialize
+	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
+
 .PHONY: fast_serialize
 fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
 	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)

cookbook/scripts/ci_helper.sh

@@ -10,6 +10,14 @@ release_example() {
   done
 }
 
+register_example() {
+  for row in $(cat flyte_tests_manifest.json | jq -c '.[]'); do
+    flytectl register file "./release-snacks/flytesnacks-$(echo ${row} | jq -r '.name').tgz" -p flytesnacks -d development --archive
+  done
+}
+
 if [ "$1" == "RELEASE" ]; then
   release_example
+elif [ $1 == "REGISTER" ]; then
+  register_example
 fi