From c6a765332aaf7d7bf2ffc1a6d85aaae6f4ec3068 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 7 Dec 2023 17:27:29 +0530 Subject: [PATCH] remove hardcoded list of tests (#4521) * remove hardcoded list of tests Signed-off-by: Samhita Alla * latest version Signed-off-by: Samhita Alla * latest version Signed-off-by: Samhita Alla * add sensor Signed-off-by: Samhita Alla * add flytesnacks to the path Signed-off-by: Samhita Alla * add flytesnacks to the path Signed-off-by: Samhita Alla --------- Signed-off-by: Samhita Alla Co-authored-by: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> --- .github/workflows/end2end.yml | 15 +++------------ .github/workflows/single-binary.yml | 23 +++++++---------------- docs/integrations.md | 6 ++---- 3 files changed, 12 insertions(+), 32 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index cee310b195..aff4665fee 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -79,16 +79,7 @@ jobs: - name: Register P0 tests if: ${{ inputs.priorities == 'P0' }} run: | - for f in \ - basics/basics/hello_world.py \ - basics/basics/workflow.py \ - basics/basics/named_outputs.py \ - advanced_composition/advanced_composition/chain_entities.py \ - advanced_composition/advanced_composition/dynamics.py \ - advanced_composition/advanced_composition/map_task.py \ - advanced_composition/advanced_composition/subworkflows.py \ - data_types_and_io/data_types_and_io/dataclass.py \ - data_types_and_io/data_types_and_io/structured_dataset.py ; + while read -r line; do pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ register \ @@ -96,8 +87,8 @@ jobs: --domain development \ --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/examples/$f; - done + flytesnacks/$line; + done < flytesnacks/flyte_tests.txt - name: Register all flytesnacks examples if: ${{ inputs.priorities != 'P0' }} uses: unionai/flyte-register-action@v0.0.2 diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml index 630068da00..5a22cf4594 100644 --- a/.github/workflows/single-binary.yml +++ b/.github/workflows/single-binary.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: "1.19" - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: @@ -129,8 +129,8 @@ jobs: sandbox-bundled-functional-tests: runs-on: ubuntu-latest env: - FLYTESNACKS_PRIORITIES: 'P0' - FLYTESNACKS_VERSION: '' + FLYTESNACKS_PRIORITIES: "P0" + FLYTESNACKS_VERSION: "" timeout-minutes: 20 needs: [build-and-push-single-binary-image] steps: @@ -164,7 +164,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: "3.11" - uses: unionai/flytectl-setup-action@v0.0.1 - name: Setup sandbox run: | @@ -193,16 +193,7 @@ jobs: # ref: ${{ env.FLYTESNACKS_VERSION }} - name: Register specific tests run: | - for f in \ - basics/basics/hello_world.py \ - basics/basics/workflow.py \ - basics/basics/named_outputs.py \ - advanced_composition/advanced_composition/chain_entities.py \ - advanced_composition/advanced_composition/dynamics.py \ - advanced_composition/advanced_composition/map_task.py \ - advanced_composition/advanced_composition/subworkflows.py \ - data_types_and_io/data_types_and_io/dataclass.py \ - data_types_and_io/data_types_and_io/structured_dataset.py ; + while read -r line; do pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ register \ @@ -210,8 +201,8 @@ jobs: --domain development \ --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/examples/$f; - done + flytesnacks/$line; + done < flytesnacks/flyte_tests.txt - name: End2End run: | make end2end_execute diff --git a/docs/integrations.md b/docs/integrations.md index 34ebfd073d..5a4bbcc909 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -49,13 +49,12 @@ Data is automatically marshalled and unmarshalled in and out of the plugin. User {py:class}`~flytekit.core.base_task.PythonTask` API defined in Flytekit. Flytekit Plugins are lazily loaded and can be released independently like libraries. We follow a convention to name the -plugin like `flytekitplugins-*`, where * indicates the package to be integrated into Flytekit. For example +plugin like `flytekitplugins-*`, where \* indicates the package to be integrated into Flytekit. For example `flytekitplugins-papermill` enables users to author Flytekit tasks using [Papermill](https://papermill.readthedocs.io/en/latest/). You can find the plugins maintained by the core Flyte team [here](https://github.com/flyteorg/flytekit/tree/master/plugins). ::: - ## Native Backend Plugins Native Backend Plugins are the plugins that can be executed without any external service dependencies because the compute is @@ -83,7 +82,6 @@ orchestrated by Flyte itself, within its provisioned Kubernetes clusters. (external_service_backend_plugins)= - ## External Service Backend Plugins As the term suggests, external service backend plugins relies on external services like @@ -138,7 +136,6 @@ Flyte uses Kustomize to generate the the deployment configuration which can be l :::: - ## Custom Container Tasks Because Flyte uses executable docker containers as the smallest unit of compute, you can write custom tasks with the @@ -208,6 +205,7 @@ flytesnacks/examples/athena_plugin/index flytesnacks/examples/aws_batch_plugin/index flytesnacks/examples/hive_plugin/index flytesnacks/examples/mmcloud_plugin/index +flytesnacks/examples/sensor/index flytesnacks/examples/snowflake_plugin/index flytesnacks/examples/databricks_plugin/index flytesnacks/examples/bigquery_plugin/index