Fixed CI for new structure changes in repo (flyteorg#186)

* fixed ci for new changes Signed-off-by: Yuvraj <[email protected]> * Added empty line in makefile Signed-off-by: Yuvraj <[email protected]>
eapolinario · May 18, 2021 · 34cb437 · 34cb437
1 parent 1417041
commit 34cb437
Show file tree

Hide file tree

Showing 6 changed files with 56 additions and 33 deletions.
diff --git a/.github/workflows/ghcr_core_master.yml b/.github/workflows/ghcr_core_master.yml
@@ -1,6 +1,7 @@
 name: Push Core Examples
 
 on:
+  pull_request:
   create:
     tags:
       - v*

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -38,12 +38,11 @@ jobs:
       - name: Generate Serialize proto
         run: |
           make -C cookbook/core serialize
-          mkdir -p cookbook/release-snacks
-          mkdir -p /tmp/release-snacks-core/
+          mkdir -p cookbook/release-snacks /tmp/release-snacks-core/
           make -C cookbook archive
           cp -r cookbook/release-snacks/* /tmp/release-snacks-core/
-  plugin-releaser:
-    name: Release Plugins example
+  integrations-releaser:
+    name: Release Integrations example
     runs-on: ubuntu-latest
     needs: [bump-version]
     steps:
@@ -52,15 +51,14 @@ jobs:
           fetch-depth: "0"
       - uses: actions/cache@v1
         with:
-          path: /tmp/release-snacks-plugins/
+          path: /tmp/release-snacks-integrations/
           key: release-snacks-plugins
       - name: Generate Serialize proto
         run: |
-          make -C cookbook/plugins all_serialize
-          mkdir -p cookbook/release-snacks
-          mkdir -p /tmp/release-snacks-plugins/
+          make -C cookbook/integrations serialize
+          mkdir -p cookbook/release-snacks /tmp/release-snacks-integrations/
           make -C cookbook archive
-          cp -r cookbook/release-snacks/* /tmp/release-snacks-plugins/
+          cp -r cookbook/release-snacks/* /tmp/release-snacks-integrations/
   case_studies-releaser:
     name: Release Case studies example
     runs-on: ubuntu-latest
@@ -75,15 +73,14 @@ jobs:
           key: release-snacks-case-studies
       - name: Generate Serialize proto
         run: |
-          make -C cookbook/case_studies all_serialize
-          mkdir -p cookbook/release-snacks
-          mkdir -p /tmp/release-snacks-case-studies/
+          make -C cookbook/case_studies serialize
+          mkdir -p cookbook/release-snacks /tmp/release-snacks-case-studies/
           make -C cookbook archive
           cp -r cookbook/release-snacks/* /tmp/release-snacks-case-studies/
   goreleaser:
     name: Release Case studies example
     runs-on: ubuntu-latest
-    needs: [core-releaser, case_studies-releaser, plugin-releaser]
+    needs: [core-releaser, case_studies-releaser, integrations-releaser]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -98,7 +95,7 @@ jobs:
           key: release-snacks-case-studies
       - uses: actions/cache@v1
         with:
-          path: /tmp/release-snacks-plugins/
+          path: /tmp/release-snacks-integrations/
           key: release-snacks-plugins
       - uses: actions/checkout@v2
         with:
@@ -109,7 +106,7 @@ jobs:
           mkdir -p cookbook/release-snacks
           cp -r /tmp/release-snacks-core/* cookbook/release-snacks/
           cp -r /tmp/release-snacks-case-studies/* cookbook/release-snacks/
-          cp -r /tmp/release-snacks-plugins/* cookbook/release-snacks/
+          cp -r /tmp/release-snacks-integrations/* cookbook/release-snacks/
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2
         with:

diff --git a/cookbook/common/parent.mk b/cookbook/common/parent.mk
@@ -73,4 +73,4 @@ clean: ## Deletes build directories (e.g. _pb_output/)
 		echo "processing ${PWD}/$$dir"; \
 		test -f $$dir/Makefile && \
 		$(MAKE) -C $$dir clean; \
-	done
+	done
diff --git a/cookbook/core/Makefile b/cookbook/core/Makefile
@@ -1,3 +1,4 @@
 PREFIX=core
+PWD=$(shell pwd)
 include ../common/Makefile
 include ../common/leaf.mk
diff --git a/cookbook/core/in_container.mk b/cookbook/core/in_container.mk
@@ -0,0 +1,24 @@
+SERIALIZED_PB_OUTPUT_DIR := /tmp/output
+
+.PHONY: clean
+clean:
+	rm -rf $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+$(SERIALIZED_PB_OUTPUT_DIR): clean
+	mkdir -p $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: serialize
+serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: register
+register: serialize
+	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+.PHONY: fast_serialize
+fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: fast_register
+fast_register: fast_serialize
+	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/flyte_tests_manifest.json b/cookbook/flyte_tests_manifest.json
@@ -7,65 +7,65 @@
       "exit_message": ""
     }
   },{
-    "name": "Plugin-hive",
+    "name": "integrations-hive",
     "priority": "P1",
-    "path": "plugins/hive",
+    "path": "integrations/external_services/hive",
     "exitCondition": {
       "exit_success": true,
         "exit_message": ""
     }
   },{
-    "name": "plugins-pod",
+    "name": "integrations-k8s-spark",
     "priority": "P1",
-    "path": "plugins/pod",
+    "path": "integrations/kubernetes/k8s_spark",
     "exitCondition": {
       "exit_success": true,
         "exit_message": ""
     }
   },{
-    "name": "plugins-k8s-spark",
+    "name": "integrations-kfpytorch",
     "priority": "P1",
-    "path": "plugins/k8s_spark",
+    "path": "integrations/kubernetes/kfpytorch",
     "exitCondition": {
       "exit_success": true,
         "exit_message": ""
     }
   },{
-    "name": "plugins-kfpytorch",
+    "name": "integrations-pod",
     "priority": "P1",
-    "path": "plugins/kfpytorch",
+    "path": "integrations/kubernetes/pod",
     "exitCondition": {
       "exit_success": true,
         "exit_message": ""
     }
   },{
-    "name": "plugins-pandera",
+    "name": "integrations-pandera",
     "priority": "P1",
-    "path": "plugins/pandera",
+    "path": "integrations/flytekit_plugins/pandera",
     "exitCondition": {
       "exit_success": true,
         "exit_message": ""
     }
   },{
-    "name": "plugins-papermilltasks",
+    "name": "integrations-papermilltasks",
     "priority": "P1",
-    "path": "plugins/papermilltasks",
+    "path": "integrations/flytekit_plugins/papermilltasks",
     "exitCondition": {
       "exit_success": true,
         "exit_message": ""
     }
   },{
-    "name": "plugins-sagemaker-pytorch",
+    "name": "integrations-sagemaker-pytorch",
     "priority": "P1",
-    "path": "plugins/sagemaker_pytorch",
+    "path": "integrations/aws/sagemaker_pytorch",
     "exitCondition": {
       "exit_success": true,
         "exit_message": ""
     }
   },{
-    "name": "plugins-sagemaker-training",
+    "name": "integrations-sagemaker-training",
     "priority": "P1",
-    "path": "plugins/sagemaker_training",
+    "path": "integrations/aws/sagemaker_training",
     "exitCondition": {
       "exit_success": true,
         "exit_message": ""
@@ -81,7 +81,7 @@
  },{
    "name": "case-studies-pima-diabetes",
    "priority": "P2",
-   "path": "case_studies/pima-diabetes",
+   "path": "case_studies/ml_training/pima-diabetes",
    "exitCondition": {
      "exit_success": true,
      "exit_message": ""