Add boilerplate and fix flytectl (#1239)

* add boilerplate Signed-off-by: Eduardo Apolinario <[email protected]> * Fix version of flytectl Signed-off-by: Eduardo Apolinario <[email protected]> * Fix workflow name used to bump boilerplate Signed-off-by: Eduardo Apolinario <[email protected]> * lint Signed-off-by: Eduardo Apolinario <[email protected]> * fix indentation Signed-off-by: Eduardo Apolinario <[email protected]> * Use correct reference launchplan Signed-off-by: Eduardo Apolinario <[email protected]> * Pin flytekit version in e2e tests Signed-off-by: Eduardo Apolinario <[email protected]> * Use correct reference task Signed-off-by: Eduardo Apolinario <[email protected]> --------- Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
flyteorg · Nov 13, 2023 · b1e1fcd · b1e1fcd
1 parent 5f8b78c
commit b1e1fcd
Show file tree

Hide file tree

Showing 10 changed files with 81 additions and 38 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -223,11 +223,12 @@ jobs:
                   data_types_and_io/data_types_and_io/dataclass.py \
                   data_types_and_io/data_types_and_io/structured_dataset.py ;
           do
+              # TODO: unpin version of flytekit once the FlyteFile bug is fixed
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
-                  --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
+                  --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
                   flytesnacks/examples/$f;
           done

diff --git a/.github/workflows/serialize_example.yml b/.github/workflows/serialize_example.yml
@@ -53,6 +53,8 @@ jobs:
         with:
           fetch-depth: "0"
       - uses: unionai/[email protected]
+        with:
+          version: 'v0.7.8'
       - name: setup download artifact dir
         run: |
           mkdir download-artifact

diff --git a/.github/workflows/upgrade_automtion.yml → .github/workflows/upgrade_automation.yml b/.github/workflows/upgrade_automtion.yml → .github/workflows/upgrade_automation.yml
diff --git a/Makefile b/Makefile
@@ -1,3 +1,4 @@
+export REPOSITORY=flytesnacks
 include boilerplate/flyte/end2end/Makefile
 .SILENT:
 
@@ -27,3 +28,7 @@ fmt: ## Format code with black and isort
 	autoflake --remove-all-unused-imports --ignore-init-module-imports --ignore-pass-after-docstring --in-place -r examples
 	pre-commit run black --all-files || true
 	pre-commit run isort --all-files || true
+
+.PHONY: update_boilerplate
+update_boilerplate:
+	@boilerplate/update.sh
diff --git a/boilerplate/flyte/code_of_conduct/update.sh b/boilerplate/flyte/code_of_conduct/update.sh
@@ -7,6 +7,6 @@
 
 set -e
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
 
-cp "${DIR}"/CODE_OF_CONDUCT.md "${DIR}"/../../../CODE_OF_CONDUCT.md
+cp "$DIR"/CODE_OF_CONDUCT.md "$DIR"/../../../CODE_OF_CONDUCT.md
diff --git a/boilerplate/flyte/end2end/end2end.sh b/boilerplate/flyte/end2end/end2end.sh
@@ -6,7 +6,8 @@
 # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
 set -eu
 
-CONFIG_FILE=$1; shift
-EXTRA_FLAGS=( "$@" )
+CONFIG_FILE=$1
+shift
+EXTRA_FLAGS=("$@")
 
 python ./boilerplate/flyte/end2end/run-tests.py "$FLYTESNACKS_VERSION" "$FLYTESNACKS_PRIORITIES" "$CONFIG_FILE" "${EXTRA_FLAGS[@]}"
diff --git a/boilerplate/flyte/end2end/functional-test-config.yaml b/boilerplate/flyte/end2end/functional-test-config.yaml
@@ -1,18 +1,5 @@
 admin:
+  # For GRPC endpoints you might want to use dns:///flyte.myexample.com
   endpoint: dns:///localhost:30080
-  insecure: true
   authType: Pkce
-logger:
-  show-source: true
-  level: 0
-storage:
-  connection:
-    access-key: minio
-    auth-type: accesskey
-    disable-ssl: true
-    endpoint: http://localhost:30002
-    region: us-east-1
-    secret-key: miniostorage
-  type: minio
-  container: "my-s3-bucket"
-  enable-multicontainer: true
+  insecure: true
diff --git a/boilerplate/flyte/end2end/run-tests.py b/boilerplate/flyte/end2end/run-tests.py
@@ -5,7 +5,7 @@
 import sys
 import time
 import traceback
-from typing import Dict, List, Mapping, Tuple
+from typing import Dict, List, Mapping, Optional, Tuple
 
 import click
 import requests
@@ -50,15 +50,17 @@
         ("basics.named_outputs.simple_wf_with_named_outputs", {}),
         # # Getting a 403 for the wikipedia image
         # # ("basics.reference_task.wf", {}),
-        ("data_types_and_io.custom_objects.wf", {"x": 10, "y": 20}),
+        ("data_types_and_io.dataclass.dataclass_wf", {"x": 10, "y": 20}),
         # Enums are not supported in flyteremote
         # ("type_system.enums.enum_wf", {"c": "red"}),
-        ("data_types_and_io.schema.df_wf", {"a": 42}),
-        ("data_types_and_io.typed_schema.wf", {}),
+        ("data_types_and_io.structured_dataset.simple_sd_wf", {"a": 42}),
         # ("my.imperative.workflow.example", {"in1": "hello", "in2": "foo"}),
     ],
     "integrations-k8s-spark": [
-        ("k8s_spark_plugin.pyspark_pi.my_spark", {"triggered_date": datetime.datetime.now()}),
+        (
+            "k8s_spark_plugin.pyspark_pi.my_spark",
+            {"triggered_date": datetime.datetime.now()},
+        ),
     ],
     "integrations-kfpytorch": [
         ("kfpytorch_plugin.pytorch_mnist.pytorch_training_wf", {}),
@@ -89,10 +91,16 @@
 }
 
 
-def execute_workflow(remote, version, workflow_name, inputs):
+def execute_workflow(
+    remote: FlyteRemote,
+    version,
+    workflow_name,
+    inputs,
+    cluster_pool_name: Optional[str] = None,
+):
     print(f"Fetching workflow={workflow_name} and version={version}")
     wf = remote.fetch_workflow(name=workflow_name, version=version)
-    return remote.execute(wf, inputs=inputs, wait=False)
+    return remote.execute(wf, inputs=inputs, wait=False, cluster_pool=cluster_pool_name)
 
 
 def executions_finished(executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]) -> bool:
@@ -125,17 +133,18 @@ def schedule_workflow_groups(
     workflow_groups: List[str],
     remote: FlyteRemote,
     terminate_workflow_on_failure: bool,
+    cluster_pool_name: Optional[str] = None,
 ) -> Dict[str, bool]:
     """
-    Schedule workflows executions for all workflow gropus and return True if all executions succeed, otherwise
+    Schedule workflows executions for all workflow groups and return True if all executions succeed, otherwise
     return False.
     """
     executions_by_wfgroup = {}
     # Schedule executions for each workflow group,
     for wf_group in workflow_groups:
         workflows = FLYTESNACKS_WORKFLOW_GROUPS.get(wf_group, [])
         executions_by_wfgroup[wf_group] = [
-            execute_workflow(remote, tag, workflow[0], workflow[1]) for workflow in workflows
+            execute_workflow(remote, tag, workflow[0], workflow[1], cluster_pool_name) for workflow in workflows
         ]
 
     # Wait for all executions to finish
@@ -179,11 +188,14 @@ def run(
     priorities: List[str],
     config_file_path,
     terminate_workflow_on_failure: bool,
+    test_project_name: str,
+    test_project_domain: str,
+    cluster_pool_name: Optional[str] = None,
 ) -> List[Dict[str, str]]:
     remote = FlyteRemote(
         Config.auto(config_file=config_file_path),
-        default_project="flytesnacks",
-        default_domain="development",
+        test_project_name,
+        test_project_domain,
     )
 
     # For a given release tag and priority, this function filters the workflow groups from the flytesnacks
@@ -215,7 +227,11 @@ def run(
         valid_workgroups.append(workflow_group)
 
     results_by_wfgroup = schedule_workflow_groups(
-        flytesnacks_release_tag, valid_workgroups, remote, terminate_workflow_on_failure
+        flytesnacks_release_tag,
+        valid_workgroups,
+        remote,
+        terminate_workflow_on_failure,
+        cluster_pool_name,
     )
 
     for workflow_group, succeeded in results_by_wfgroup.items():
@@ -246,6 +262,9 @@ def run(
 
 
 @click.command()
+@click.argument("flytesnacks_release_tag")
+@click.argument("priorities")
+@click.argument("config_file")
 @click.option(
     "--return_non_zero_on_failure",
     default=False,
@@ -258,18 +277,46 @@ def run(
     is_flag=True,
     help="Abort failing workflows upon exit",
 )
-@click.argument("flytesnacks_release_tag")
-@click.argument("priorities")
-@click.argument("config_file")
+@click.option(
+    "--test_project_name",
+    default="flytesnacks",
+    type=str,
+    is_flag=False,
+    help="Name of project to run functional tests on",
+)
+@click.option(
+    "--test_project_domain",
+    default="development",
+    type=str,
+    is_flag=False,
+    help="Name of domain in project to run functional tests on",
+)
+@click.argument(
+    "cluster_pool_name",
+    required=False,
+    type=str,
+    default=None,
+)
 def cli(
     flytesnacks_release_tag,
     priorities,
     config_file,
     return_non_zero_on_failure,
     terminate_workflow_on_failure,
+    test_project_name,
+    test_project_domain,
+    cluster_pool_name,
 ):
     print(f"return_non_zero_on_failure={return_non_zero_on_failure}")
-    results = run(flytesnacks_release_tag, priorities, config_file, terminate_workflow_on_failure)
+    results = run(
+        flytesnacks_release_tag,
+        priorities,
+        config_file,
+        terminate_workflow_on_failure,
+        test_project_name,
+        test_project_domain,
+        cluster_pool_name,
+    )
 
     # Write a json object in its own line describing the result of this run to stdout
     print(f"Result of run:\n{json.dumps(results)}")

diff --git a/examples/productionizing/productionizing/reference_launch_plan.py b/examples/productionizing/productionizing/reference_launch_plan.py
@@ -24,7 +24,7 @@
 @reference_launch_plan(
     project="flytesnacks",
     domain="development",
-    name="advanced_composition.files.normalize_csv_file",
+    name="data_types_and_io.file.normalize_csv_file",
     version="{{ registration.version }}",
 )
 def normalize_csv_file(

diff --git a/examples/productionizing/productionizing/reference_task.py b/examples/productionizing/productionizing/reference_task.py
@@ -24,7 +24,7 @@
 @reference_task(
     project="flytesnacks",
     domain="development",
-    name="advanced_composition.files.normalize_columns",
+    name="data_types_and_io.file.normalize_columns",
     version="{{ registration.version }}",
 )
 def normalize_columns(