From 3966f5207d61ee083410fb998f44c1c68e0703b1 Mon Sep 17 00:00:00 2001 From: Johnny Burns Date: Mon, 21 Oct 2019 10:41:28 -0700 Subject: [PATCH] Initial Commit --- .gitattributes | 1 + .gitignore | 14 + .travis.yml | 11 + CODE_OF_CONDUCT.md | 3 + LICENSE | 202 +++ Makefile | 32 + NOTICE | 4 + README.md | 5 + deployment/sandbox/flyte_generated.yaml | 1545 +++++++++++++++++ deployment/test/flyte_generated.yaml | 1101 ++++++++++++ docs_infra/in_container_html_generation.sh | 18 + docs_infra/in_container_rst_generation.sh | 25 + docs_infra/index.rst | 22 + end2end/Readme.rst | 26 + end2end/dockernetes_run.sh | 21 + end2end/execute.sh | 61 + end2end/launch_dockernetes.sh | 27 + end2end/print_logs.sh | 17 + end2end/test_monitor.sh | 34 + end2end/tests/endtoend.yaml | 23 + .../base/admindeployment/deployment.yaml | 70 + .../base/admindeployment/kustomization.yaml | 3 + kustomize/base/admindeployment/service.yaml | 25 + .../adminserviceaccount.yaml | 49 + .../adminserviceaccount/kustomization.yaml | 2 + kustomize/base/console/configmap.yaml | 10 + kustomize/base/console/deployment.yaml | 33 + kustomize/base/console/kustomization.yaml | 4 + kustomize/base/console/service.yaml | 14 + kustomize/base/datacatalog/deployment.yaml | 47 + kustomize/base/datacatalog/kustomization.yaml | 3 + kustomize/base/datacatalog/service.yaml | 24 + kustomize/base/ingress/ingress.yaml | 44 + kustomize/base/ingress/kustomization.yaml | 2 + kustomize/base/namespace/kustomization.yaml | 2 + kustomize/base/namespace/namespace.yaml | 10 + .../base/operators/spark/deployment.yaml | 44 + .../base/operators/spark/kustomization.yaml | 4 + .../base/operators/spark/spark-dummy-crd.yaml | 15 + .../base/operators/spark/spark-operator.yaml | 56 + kustomize/base/propeller/deployment.yaml | 52 + kustomize/base/propeller/kustomization.yaml | 3 + kustomize/base/propeller/rbac.yaml | 88 + kustomize/base/wf_crd/kustomization.yaml | 2 + kustomize/base/wf_crd/wf_crd.yaml | 22 + .../custom-config.yaml | 134 ++ .../deployment.yaml | 66 + .../kustomization.yaml | 6 + .../contour_ingress_controller/namespace.yaml | 4 + .../contour_ingress_controller/rbac.yaml | 66 + .../contour_ingress_controller/service.yaml | 17 + kustomize/dependencies/database/database.yaml | 38 + .../dependencies/database/kustomization.yaml | 2 + kustomize/dependencies/redis/deployment.yaml | 56 + .../dependencies/redis/kustomization.yaml | 3 + kustomize/dependencies/redis/service.yaml | 16 + .../dependencies/storage/kustomization.yaml | 2 + kustomize/dependencies/storage/storage.yaml | 63 + .../admindeployment/admindeployment.yaml | 49 + .../aa_namespace.yaml | 7 + .../sandbox/admindeployment/cron.yaml | 29 + .../admindeployment/flyteadmin_config.yaml | 102 ++ .../admindeployment/kustomization.yaml | 21 + .../overlays/sandbox/console/console.yaml | 15 + .../sandbox/console/kustomization.yaml | 5 + .../sandbox/datacatalog/datacatalog.yaml | 30 + .../datacatalog/datacatalog_config.yaml | 28 + .../sandbox/datacatalog/kustomization.yaml | 12 + .../overlays/sandbox/flyte/kustomization.yaml | 19 + .../overlays/sandbox/propeller/config.yaml | 54 + .../sandbox/propeller/kustomization.yaml | 35 + .../sandbox/propeller/plugins/config.yaml | 15 + .../propeller/plugins/container/config.yaml | 0 .../plugins/container/propeller-patch.yaml | 17 + .../propeller/plugins/qubole/config.yaml | 10 + .../plugins/qubole/propeller-patch.yaml | 20 + .../propeller/plugins/spark/config.yaml | 16 + .../plugins/spark/propeller-patch.yaml | 17 + .../propeller/plugins/waitable/config.yaml | 3 + .../plugins/waitable/propeller-patch.yaml | 17 + .../overlays/sandbox/propeller/propeller.yaml | 18 + .../overlays/sandbox/redis/kustomization.yaml | 5 + kustomize/overlays/sandbox/redis/storage.yaml | 11 + .../overlays/test/flyte/kustomization.yaml | 13 + kustomize/overlays/test/propeller/config.yaml | 50 + .../test/propeller/kustomization.yaml | 17 + .../test/propeller/plugins/config.yaml | 12 + .../overlays/test/propeller/propeller.yaml | 15 + .../overlays/test/redis/kustomization.yaml | 5 + kustomize/overlays/test/redis/storage.yaml | 11 + rsts/administrator/architecture.rst | 93 + rsts/administrator/index.rst | 17 + .../administrator/install/configure/admin.rst | 12 + .../install/configure/common.rst | 59 + .../administrator/install/configure/index.rst | 15 + .../install/configure/plugins.rst | 73 + .../install/configure/propeller.rst | 195 +++ .../administrator/install/getting_started.rst | 46 + rsts/administrator/install/index.rst | 16 + rsts/administrator/install/multi_cluster.rst | 123 ++ .../install/optional_components.rst | 17 + rsts/administrator/install/production.rst | 156 ++ rsts/conf.py | 186 ++ rsts/contributor/components/admin.rst | 343 ++++ rsts/contributor/components/catalog.rst | 61 + rsts/contributor/components/console.rst | 124 ++ rsts/contributor/components/index.rst | 13 + rsts/contributor/docs/index.rst | 36 + rsts/contributor/extending/index.rst | 81 + rsts/contributor/index.rst | 18 + rsts/contributor/language/index.rst | 200 +++ rsts/index.rst | 23 + rsts/introduction/docs_overview.rst | 47 + rsts/introduction/index.rst | 14 + rsts/introduction/whatis.rst | 34 + rsts/user/concepts/domains.rst | 11 + rsts/user/concepts/dynamic_spec.rst | 41 + rsts/user/concepts/execution_timeline.rst | 7 + rsts/user/concepts/executions.rst | 16 + rsts/user/concepts/index.rst | 49 + rsts/user/concepts/launchplans_schedules.rst | 60 + rsts/user/concepts/projects.rst | 7 + rsts/user/concepts/registration.rst | 35 + rsts/user/concepts/tasks.rst | 82 + rsts/user/concepts/workflows_nodes.rst | 66 + rsts/user/features/flytecli.rst | 151 ++ rsts/user/features/index.rst | 16 + rsts/user/features/labels_annotations.rst | 53 + rsts/user/features/lanuchplans.rst | 131 ++ rsts/user/features/notifications.rst | 30 + rsts/user/features/observability.rst | 76 + rsts/user/features/roles.rst | 43 + rsts/user/features/task_cache.rst | 34 + rsts/user/getting_started/create_first.rst | 122 ++ rsts/user/getting_started/examples.rst | 50 + rsts/user/getting_started/index.rst | 16 + rsts/user/index.rst | 17 + rsts/user/sdk/index.rst | 8 + rsts/user/tasktypes/container.rst | 35 + rsts/user/tasktypes/dynamic.rst | 49 + rsts/user/tasktypes/hive.rst | 80 + rsts/user/tasktypes/index.rst | 15 + rsts/user/tasktypes/sidecar.rst | 138 ++ rsts/user/tasktypes/spark.rst | 51 + script/deploy.sh | 6 + script/flyte_venv.sh | 10 + script/generate_docs.sh | 29 + script/kustomize.sh | 10 + script/update_ref_docs.sh | 27 + 149 files changed, 8544 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 CODE_OF_CONDUCT.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 NOTICE create mode 100644 README.md create mode 100644 deployment/sandbox/flyte_generated.yaml create mode 100644 deployment/test/flyte_generated.yaml create mode 100755 docs_infra/in_container_html_generation.sh create mode 100755 docs_infra/in_container_rst_generation.sh create mode 100644 docs_infra/index.rst create mode 100644 end2end/Readme.rst create mode 100755 end2end/dockernetes_run.sh create mode 100755 end2end/execute.sh create mode 100755 end2end/launch_dockernetes.sh create mode 100755 end2end/print_logs.sh create mode 100755 end2end/test_monitor.sh create mode 100644 end2end/tests/endtoend.yaml create mode 100644 kustomize/base/admindeployment/deployment.yaml create mode 100644 kustomize/base/admindeployment/kustomization.yaml create mode 100644 kustomize/base/admindeployment/service.yaml create mode 100644 kustomize/base/adminserviceaccount/adminserviceaccount.yaml create mode 100644 kustomize/base/adminserviceaccount/kustomization.yaml create mode 100644 kustomize/base/console/configmap.yaml create mode 100644 kustomize/base/console/deployment.yaml create mode 100644 kustomize/base/console/kustomization.yaml create mode 100644 kustomize/base/console/service.yaml create mode 100644 kustomize/base/datacatalog/deployment.yaml create mode 100644 kustomize/base/datacatalog/kustomization.yaml create mode 100644 kustomize/base/datacatalog/service.yaml create mode 100644 kustomize/base/ingress/ingress.yaml create mode 100644 kustomize/base/ingress/kustomization.yaml create mode 100644 kustomize/base/namespace/kustomization.yaml create mode 100644 kustomize/base/namespace/namespace.yaml create mode 100644 kustomize/base/operators/spark/deployment.yaml create mode 100644 kustomize/base/operators/spark/kustomization.yaml create mode 100644 kustomize/base/operators/spark/spark-dummy-crd.yaml create mode 100644 kustomize/base/operators/spark/spark-operator.yaml create mode 100644 kustomize/base/propeller/deployment.yaml create mode 100644 kustomize/base/propeller/kustomization.yaml create mode 100644 kustomize/base/propeller/rbac.yaml create mode 100644 kustomize/base/wf_crd/kustomization.yaml create mode 100644 kustomize/base/wf_crd/wf_crd.yaml create mode 100644 kustomize/dependencies/contour_ingress_controller/custom-config.yaml create mode 100644 kustomize/dependencies/contour_ingress_controller/deployment.yaml create mode 100644 kustomize/dependencies/contour_ingress_controller/kustomization.yaml create mode 100644 kustomize/dependencies/contour_ingress_controller/namespace.yaml create mode 100644 kustomize/dependencies/contour_ingress_controller/rbac.yaml create mode 100644 kustomize/dependencies/contour_ingress_controller/service.yaml create mode 100644 kustomize/dependencies/database/database.yaml create mode 100644 kustomize/dependencies/database/kustomization.yaml create mode 100644 kustomize/dependencies/redis/deployment.yaml create mode 100644 kustomize/dependencies/redis/kustomization.yaml create mode 100644 kustomize/dependencies/redis/service.yaml create mode 100644 kustomize/dependencies/storage/kustomization.yaml create mode 100644 kustomize/dependencies/storage/storage.yaml create mode 100644 kustomize/overlays/sandbox/admindeployment/admindeployment.yaml create mode 100644 kustomize/overlays/sandbox/admindeployment/clusterresource-templates/aa_namespace.yaml create mode 100644 kustomize/overlays/sandbox/admindeployment/cron.yaml create mode 100644 kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml create mode 100644 kustomize/overlays/sandbox/admindeployment/kustomization.yaml create mode 100644 kustomize/overlays/sandbox/console/console.yaml create mode 100644 kustomize/overlays/sandbox/console/kustomization.yaml create mode 100644 kustomize/overlays/sandbox/datacatalog/datacatalog.yaml create mode 100644 kustomize/overlays/sandbox/datacatalog/datacatalog_config.yaml create mode 100644 kustomize/overlays/sandbox/datacatalog/kustomization.yaml create mode 100644 kustomize/overlays/sandbox/flyte/kustomization.yaml create mode 100644 kustomize/overlays/sandbox/propeller/config.yaml create mode 100644 kustomize/overlays/sandbox/propeller/kustomization.yaml create mode 100644 kustomize/overlays/sandbox/propeller/plugins/config.yaml create mode 100644 kustomize/overlays/sandbox/propeller/plugins/container/config.yaml create mode 100644 kustomize/overlays/sandbox/propeller/plugins/container/propeller-patch.yaml create mode 100644 kustomize/overlays/sandbox/propeller/plugins/qubole/config.yaml create mode 100644 kustomize/overlays/sandbox/propeller/plugins/qubole/propeller-patch.yaml create mode 100644 kustomize/overlays/sandbox/propeller/plugins/spark/config.yaml create mode 100644 kustomize/overlays/sandbox/propeller/plugins/spark/propeller-patch.yaml create mode 100644 kustomize/overlays/sandbox/propeller/plugins/waitable/config.yaml create mode 100644 kustomize/overlays/sandbox/propeller/plugins/waitable/propeller-patch.yaml create mode 100644 kustomize/overlays/sandbox/propeller/propeller.yaml create mode 100644 kustomize/overlays/sandbox/redis/kustomization.yaml create mode 100644 kustomize/overlays/sandbox/redis/storage.yaml create mode 100644 kustomize/overlays/test/flyte/kustomization.yaml create mode 100644 kustomize/overlays/test/propeller/config.yaml create mode 100644 kustomize/overlays/test/propeller/kustomization.yaml create mode 100644 kustomize/overlays/test/propeller/plugins/config.yaml create mode 100644 kustomize/overlays/test/propeller/propeller.yaml create mode 100644 kustomize/overlays/test/redis/kustomization.yaml create mode 100644 kustomize/overlays/test/redis/storage.yaml create mode 100644 rsts/administrator/architecture.rst create mode 100644 rsts/administrator/index.rst create mode 100644 rsts/administrator/install/configure/admin.rst create mode 100644 rsts/administrator/install/configure/common.rst create mode 100644 rsts/administrator/install/configure/index.rst create mode 100644 rsts/administrator/install/configure/plugins.rst create mode 100644 rsts/administrator/install/configure/propeller.rst create mode 100644 rsts/administrator/install/getting_started.rst create mode 100644 rsts/administrator/install/index.rst create mode 100644 rsts/administrator/install/multi_cluster.rst create mode 100644 rsts/administrator/install/optional_components.rst create mode 100644 rsts/administrator/install/production.rst create mode 100644 rsts/conf.py create mode 100644 rsts/contributor/components/admin.rst create mode 100644 rsts/contributor/components/catalog.rst create mode 100644 rsts/contributor/components/console.rst create mode 100644 rsts/contributor/components/index.rst create mode 100644 rsts/contributor/docs/index.rst create mode 100644 rsts/contributor/extending/index.rst create mode 100644 rsts/contributor/index.rst create mode 100644 rsts/contributor/language/index.rst create mode 100644 rsts/index.rst create mode 100644 rsts/introduction/docs_overview.rst create mode 100644 rsts/introduction/index.rst create mode 100644 rsts/introduction/whatis.rst create mode 100644 rsts/user/concepts/domains.rst create mode 100644 rsts/user/concepts/dynamic_spec.rst create mode 100644 rsts/user/concepts/execution_timeline.rst create mode 100644 rsts/user/concepts/executions.rst create mode 100644 rsts/user/concepts/index.rst create mode 100644 rsts/user/concepts/launchplans_schedules.rst create mode 100644 rsts/user/concepts/projects.rst create mode 100644 rsts/user/concepts/registration.rst create mode 100644 rsts/user/concepts/tasks.rst create mode 100644 rsts/user/concepts/workflows_nodes.rst create mode 100644 rsts/user/features/flytecli.rst create mode 100644 rsts/user/features/index.rst create mode 100644 rsts/user/features/labels_annotations.rst create mode 100644 rsts/user/features/lanuchplans.rst create mode 100644 rsts/user/features/notifications.rst create mode 100644 rsts/user/features/observability.rst create mode 100644 rsts/user/features/roles.rst create mode 100644 rsts/user/features/task_cache.rst create mode 100644 rsts/user/getting_started/create_first.rst create mode 100644 rsts/user/getting_started/examples.rst create mode 100644 rsts/user/getting_started/index.rst create mode 100644 rsts/user/index.rst create mode 100644 rsts/user/sdk/index.rst create mode 100644 rsts/user/tasktypes/container.rst create mode 100644 rsts/user/tasktypes/dynamic.rst create mode 100644 rsts/user/tasktypes/hive.rst create mode 100644 rsts/user/tasktypes/index.rst create mode 100644 rsts/user/tasktypes/sidecar.rst create mode 100644 rsts/user/tasktypes/spark.rst create mode 100755 script/deploy.sh create mode 100755 script/flyte_venv.sh create mode 100755 script/generate_docs.sh create mode 100755 script/kustomize.sh create mode 100755 script/update_ref_docs.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..e4b260b693 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +docs/**/*html linguist-generated=true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..d8bf5924e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +.idea +_build/ +build/ +.vscode/ +*.swp +*.swo +.DS_Store +_repos/ +_rsts/ +rsts_tmp/ +.doctrees/ +docs/_sources/ +docs/flytekit/flytekit.interfaces.html +docs/searchindex.js diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..096fa43c4c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +sudo: required +language: go +go: +- "1.10" +services: +- docker +jobs: + include: + - stage: test + name: End-to-end test + script: make end2end diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..803d8a77f3 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +This project is governed by [Lyft's code of +conduct](https://github.com/lyft/code-of-conduct). All contributors +and participants agree to abide by its terms. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..bed437514f --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 Lyft, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..2d58e0506b --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +.PHONY: kustomize +kustomize: + bash script/kustomize.sh + +.PHONY: deploy_sandbox +deploy_sandbox: + bash script/deploy.sh + +# launch dockernetes and execute tests +.PHONY: end2end +end2end: + @end2end/launch_dockernetes.sh + +# execute tests in the current kubernetes context +.PHONY: end2end_execute +end2end_execute: + @end2end/execute.sh + +# Use this target to build the rsts directory only. In order to build the entire flyte docs, use update_ref_docs && all_docs +.PHONY: docs +docs: + @docker run -t -v `pwd`:/base lyft/docbuilder:v2.2.0 sphinx-build -E -b html /base/rsts/. /base/_build + +# Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir +.PHONY: all_docs +all_docs: + @./script/generate_docs.sh + +# updates referenced docs from other repositories (e.g. flyteidl, flytekit) +.PHONY: update_ref_docs +update_ref_docs: + @./script/update_ref_docs.sh diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000..43f19fee61 --- /dev/null +++ b/NOTICE @@ -0,0 +1,4 @@ +flyte +Copyright 2019 Lyft Inc. + +This product includes software developed at Lyft Inc. diff --git a/README.md b/README.md new file mode 100644 index 0000000000..3951906403 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# Flyte + +Flyte is a K8s-native, multi-tenant, distributed orchestration engine that manages highly scalable compute workflows. Extensibility is provided by Flyte plugins and K8s operators, including executors for Spark, Hive, AWS Batch, in addition to raw K8s compute. Flyte can orchestrate arbitrary containers at scale, providing feature fanout, automatic task parallelization, strongly-typed paramerter passing, automatic cluster scale up and scale down, cluster fail-over, and fine-grained programmatic control over retry policies, error handling, and notifications. + +See docs at https://lyft.github.io/flyte/ diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml new file mode 100644 index 0000000000..e28eb1159b --- /dev/null +++ b/deployment/sandbox/flyte_generated.yaml @@ -0,0 +1,1545 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flyte +spec: + finalizers: + - kubernetes +status: + phase: Active +--- +apiVersion: v1 +kind: Namespace +metadata: + name: heptio-contour +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sparkoperator +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: flyteworkflows.flyte.lyft.com +spec: + group: flyte.lyft.com + names: + kind: FlyteWorkflow + plural: flyteworkflows + shortNames: + - fly + singular: flyteworkflow + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + labels: + component: ingressroute + name: ingressroutes.contour.heptio.com +spec: + additionalPrinterColumns: + - JSONPath: .spec.virtualhost.fqdn + description: Fully qualified domain name + name: FQDN + type: string + - JSONPath: .spec.virtualhost.tls.secretName + description: Secret with TLS credentials + name: TLS Secret + type: string + - JSONPath: .spec.routes[0].match + description: First routes defined + name: First route + type: string + - JSONPath: .status.currentStatus + description: The current status of the IngressRoute + name: Status + type: string + - JSONPath: .status.description + description: Description of the current status + name: Status Description + type: string + group: contour.heptio.com + names: + kind: IngressRoute + plural: ingressroutes + scope: Namespaced + validation: + openAPIV3Schema: + properties: + spec: + properties: + healthCheck: + properties: + healthyThresholdCount: + type: integer + intervalSeconds: + type: integer + path: + pattern: ^\/.*$ + type: string + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + required: + - path + type: object + routes: + items: + properties: + delegate: + properties: + name: + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + namespace: + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + match: + pattern: ^\/.*$ + type: string + services: + items: + properties: + healthCheck: + properties: + healthyThresholdCount: + type: integer + intervalSeconds: + type: integer + path: + pattern: ^\/.*$ + type: string + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + required: + - path + type: object + name: + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + type: integer + strategy: + enum: + - RoundRobin + - WeightedLeastRequest + - Random + - RingHash + - Maglev + type: string + weight: + type: integer + required: + - name + - port + type: object + type: array + required: + - match + type: array + strategy: + enum: + - RoundRobin + - LeastRequest + - Random + - RingHash + - Maglev + type: string + virtualhost: + properties: + aliases: + items: + pattern: ^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$ + type: string + type: array + fqdn: + pattern: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-z]{2,}$ + type: string + required: + - routes + version: v1beta1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: sparkapplications.sparkoperator.k8s.io +spec: + group: sparkoperator.k8s.io + names: + kind: SparkApplication + listKind: SparkApplicationList + plural: sparkapplications + shortNames: + - sparkapp + singular: sparkapplication + scope: Namespaced + version: v1beta1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyteadmin + namespace: flyte +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flytepropeller + namespace: flyte +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: contour + namespace: heptio-contour +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sparkoperator + namespace: sparkoperator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyteadmin + namespace: flyte +rules: +- apiGroups: + - "" + - flyte.lyft.com + - rbac.authorization.k8s.io + resources: + - configmaps + - flyteworkflows + - namespaces + - pods + - roles + - rolebindings + - secrets + - services + - serviceaccounts + - spark-role + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flytepropeller +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - delete + - patch +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - flyte.lyft.com + resources: + - flyteworkflows + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - post + - deletecollection +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: contour +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - contour.heptio.com + resources: + - ingressroutes + verbs: + - get + - list + - watch + - put + - post + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: sparkoperator +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - '*' +- apiGroups: + - "" + resources: + - services + - configmaps + verbs: + - create + - get + - delete +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - create + - get + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - get + - update + - delete + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - update + - delete +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - get + - update + - delete +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + - scheduledsparkapplications + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flyteadmin-binding + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyteadmin +subjects: +- kind: ServiceAccount + name: flyteadmin + namespace: flyte +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flytepropeller + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flytepropeller +subjects: +- kind: ServiceAccount + name: flytepropeller + namespace: flyte +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: contour +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: contour +subjects: +- kind: ServiceAccount + name: contour + namespace: heptio-contour +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: sparkoperator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sparkoperator +subjects: +- kind: ServiceAccount + name: sparkoperator + namespace: sparkoperator +--- +apiVersion: v1 +data: + aa_namespace.yaml: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes +kind: ConfigMap +metadata: + name: clusterresource-template-dgc9fcm2kh + namespace: flyte +--- +apiVersion: v1 +data: + datacatalog_config.yaml: | + logger: + show-source: true + level: 5 + datacatalog: + storage-prefix: metadata/datacatalog + metrics-scope: "datacatalog" + profiler-port: 10254 + application: + grpcPort: 8089 + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + cache: + max_size_mbs: 10 + target_gc_percent: 100 + container: my-container + type: minio + database: + port: 5432 + username: postgres + host: postgres + dbname: datacatalog + options: sslmode=disable +kind: ConfigMap +metadata: + name: datacatalog-config-5g5bbg2k49 + namespace: flyte +--- +apiVersion: v1 +data: + flyteadmin_config.yaml: | + logger: + show-source: true + level: 5 + application: + httpPort: 8088 + grpcPort: 8089 + flyteadmin: + roleNameKey: "iam.amazonaws.com/role" + profilerPort: 10254 + metricsScope: "flyte:" + metadataStoragePrefix: + - "metadata" + - "admin" + testing: + host: http://flyteadmin + database: + port: 5432 + username: postgres + host: postgres + dbname: postgres + options: sslmode=disable + scheduler: + eventScheduler: + scheme: "local" + region: "us-east-1" + scheduleRole: "arn:aws:iam::abc123:role/my-schedule-role" + targetName: "arn:aws:sqs:my-region:abc123:my-queue" + workflowExecutor: + scheme: "local" + region: "us-east-1" + scheduleQueueName: "won't-work-locally" + accountId: "173840052742" + remoteData: + region: "us-east-1" + scheme: "local" + signedUrls: + durationMinutes: 3 + notifications: + type: local + region: "us-east-1" + publisher: + topicName: "foo" + processor: + queueName: "queue" + accountId: "bar" + queues: + executionQueues: + - parent: "FlyteNonProductionTaskCPU-staging-iad" + child: "FlyteNonProductionSubTaskCPU-staging-iad" + attributes: + - all + workflowConfigs: + - project: flytekit + tags: + - all + - project: "flytetester" + domain: "development" + workflowName: "app-workflows-rich-workflow-rich-workflow" + tags: + - all + - project: "flytetester" + domain: "development" + workflowName: "app-workflows-batch-workflow-batch-tasks-workflow" + tags: + - all + - tags: + - all + storage: + type: minio + connection: + access-key: minio + auth-type: accesskey + secret-key: miniostorage + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + container: "my-s3-bucket" + task_resources: + defaults: + cpu: 100m + memory: 100Mi + storage: 5Mi + limits: + cpu: 2 + memory: 2G + storage: 20Mi + task_type_whitelist: + spark: + - project: flytekit + - project: flytetester + domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + - id: domain + name: domain + cluster_resources: + templatePath: "/etc/flyte/clusterresource/templates" + refresh: 5m +kind: ConfigMap +metadata: + name: flyte-admin-config-4gbbgmdf6d + namespace: flyte +--- +apiVersion: v1 +data: + BASE_URL: /console + CONFIG_DIR: /etc/flyte/config +kind: ConfigMap +metadata: + name: flyte-console-config + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: "" +kind: ConfigMap +metadata: + name: flyte-container-config-5k78b9cm42 + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + plugins: + enabled-plugins: + - container + - spark + - waitable + - hiveExecutor + - sidecar + logs: + kubernetes-enabled: true + kubernetes-url: "http://localhost:30082" + k8s: + default-env-vars: + - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" + - FLYTE_AWS_ACCESS_KEY_ID: minio + - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage +kind: ConfigMap +metadata: + name: flyte-plugin-config-mkhh79kf6f + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + propeller: + metadata-prefix: metadata/propeller + workers: 4 + max-workflow-retries: 30 + workflow-reeval-duration: 30s + downstream-eval-duration: 30s + limit-namespace: "all" + prof-port: 10254 + metrics-prefix: flyte + enable-admin-launcher: true + leader-election: + lock-config-map: + name: propeller-leader + namespace: flyte + enabled: true + lease-duration: 15s + renew-deadline: 10s + retry-period: 2s + queue: + type: batch + batching-interval: 2s + batch-size: -1 + queue: + type: bucket + rate: 10 + capacity: 100 + sub-queue: + type: bucket + rate: 10 + capacity: 100 + logger: + show-source: true + level: 5 + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + type: minio + container: my-s3-bucket + event: + type: admin + rate: 500 + capacity: 1000 + admin: + endpoint: flyteadmin:81 + insecure: true + catalog-cache: + endpoint: datacatalog:89 + type: datacatalog + insecure: true +kind: ConfigMap +metadata: + name: flyte-propeller-config-df4bf7dg67 + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + plugins: + qubole: + # Either create this file with your username with the real token, or set the QUBOLE_API_KEY environment variable + # See the secrets_manager.go file in the plugins repo for usage. Since the dev/test deployment of + # this has a dummy QUBOLE_API_KEY env var built in, this fake path won't break anything. + quboleTokenPath: "/Path/To/QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 10 +kind: ConfigMap +metadata: + name: flyte-qubole-config-h626c9t6f7 + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + plugins: + spark: + spark-config-default: + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" + - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.blacklist.enabled: "true" + - spark.blacklist.timeout: "5m" + - spark.task.maxfailures: "8" +kind: ConfigMap +metadata: + name: flyte-spark-config-h72bmtggc2 + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + plugins: + waitable: + console-uri: http://localhost:30081/console +kind: ConfigMap +metadata: + name: flyte-waitable-config-g8gch5968h + namespace: flyte +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + contour.heptio.com/upstream-protocol.h2c: grpc + name: datacatalog + namespace: flyte +spec: + ports: + - name: http + port: 88 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 89 + protocol: TCP + targetPort: 8089 + selector: + app: datacatalog +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + contour.heptio.com/upstream-protocol.h2c: grpc + name: flyteadmin + namespace: flyte +spec: + ports: + - name: redoc + port: 87 + protocol: TCP + targetPort: 8087 + - name: http + port: 80 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 81 + protocol: TCP + targetPort: 8089 + selector: + app: flyteadmin +--- +apiVersion: v1 +kind: Service +metadata: + name: flyteconsole + namespace: flyte +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: flyteconsole +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: flyte +spec: + externalName: minio + ports: + - port: 9000 + selector: + app: minio +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: flyte +spec: + ports: + - port: 5432 + selector: + app: postgres +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: contour + name: contour + namespace: heptio-contour +spec: + ports: + - nodePort: 30081 + port: 80 + protocol: TCP + selector: + app: contour + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: datacatalog + name: datacatalog + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: datacatalog + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: datacatalog + spec: + containers: + - command: + - datacatalog + - --logtostderr + - --config + - /etc/datacatalog/config/datacatalog_config.yaml + - serve + image: docker.io/lyft/datacatalog:v0.1.1 + imagePullPolicy: IfNotPresent + name: datacatalog + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 200Mi + volumeMounts: + - mountPath: /etc/datacatalog/config + name: config-volume + initContainers: + - command: + - sh + - -c + - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep + 2; done; + image: postgres:10.1 + name: check-db-ready + volumeMounts: + - mountPath: /etc/datacatalog/config + name: config-volume + - command: + - datacatalog + - --logtostderr + - --config + - /etc/datacatalog/config/datacatalog_config.yaml + - migrate + - run + image: docker.io/lyft/datacatalog:v0.1.1 + imagePullPolicy: IfNotPresent + name: run-migrations + volumeMounts: + - mountPath: /etc/datacatalog/config + name: config-volume + serviceAccountName: datacatalog + volumes: + - emptyDir: {} + name: shared-data + - configMap: + name: datacatalog-config-5g5bbg2k49 + name: config-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flyteadmin + name: flyteadmin + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: flyteadmin + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flyteadmin + spec: + containers: + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - serve + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: flyteadmin + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 200Mi + volumeMounts: + - mountPath: /srv/flyte + name: shared-data + - mountPath: /etc/flyte/config + name: config-volume + - command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: Flyte Admin OpenAPI + - name: SPEC_URL + value: /api/v1/openapi + - name: PORT + value: "8087" + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + name: redoc + ports: + - containerPort: 8087 + resources: + limits: + cpu: "0.1" + memory: 200Mi + initContainers: + - command: + - sh + - -c + - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep + 2; done; + image: postgres:10.1 + name: check-db-ready + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - migrate + - run + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: run-migrations + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - migrate + - seed-projects + - flytesnacks + - flytetester + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: seed-projects + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + serviceAccountName: flyteadmin + volumes: + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-admin-config-4gbbgmdf6d + name: config-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flyteconsole + name: flyteconsole + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: flyteconsole + template: + metadata: + labels: + app: flyteconsole + spec: + containers: + - envFrom: + - configMapRef: + name: flyte-console-config + image: docker.io/lyft/flyteconsole:v0.1.0 + name: flyteconsole + ports: + - containerPort: 8080 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 150Mi + volumeMounts: + - mountPath: /srv/flyte + name: shared-data + volumes: + - emptyDir: {} + name: shared-data +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flytepropeller + name: flytepropeller + namespace: flyte +spec: + selector: + matchLabels: + app: flytepropeller + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flytepropeller + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/version: 0.1.9 + spec: + containers: + - args: + - --config + - /etc/flyte/config*/config.yaml + command: + - flytepropeller + env: + - name: QUBOLE_API_KEY + value: notarealkey + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: docker.io/lyft/flytepropeller:v0.1.9 + imagePullPolicy: IfNotPresent + name: flytepropeller + ports: + - containerPort: 10254 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 100Mi + volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume + - mountPath: /etc/flyte/config-container + name: container-config-volume + - mountPath: /etc/flyte/config-waitable + name: waitable-config-volume + - mountPath: /etc/flyte/config-spark + name: spark-config-volume + - mountPath: /etc/flyte/config + name: config-volume + - mountPath: /etc/flyte/config-plugin + name: plugin-config-volume + serviceAccountName: flytepropeller + volumes: + - configMap: + name: flyte-qubole-config-h626c9t6f7 + name: qubole-config-volume + - configMap: + name: flyte-container-config-5k78b9cm42 + name: container-config-volume + - configMap: + name: flyte-waitable-config-g8gch5968h + name: waitable-config-volume + - configMap: + name: flyte-spark-config-h72bmtggc2 + name: spark-config-volume + - configMap: + name: flyte-propeller-config-df4bf7dg67 + name: config-volume + - configMap: + name: flyte-plugin-config-mkhh79kf6f + name: plugin-config-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: flyte +spec: + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - args: + - server + - /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: miniostorage + image: minio/minio:RELEASE.2019-06-04T01-15-58Z + name: minio + ports: + - containerPort: 9000 + name: minio + volumeMounts: + - mountPath: /data + name: minio-storage + volumes: + - emptyDir: {} + name: minio-storage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: flyte +spec: + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - image: postgres:10.1 + name: postgres + ports: + - containerPort: 5432 + name: postgres + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgres-storage + volumes: + - emptyDir: {} + name: postgres-storage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: contour + name: contour + namespace: heptio-contour +spec: + replicas: 1 + selector: + matchLabels: + app: contour + template: + metadata: + annotations: + prometheus.io/format: prometheus + prometheus.io/path: /stats + prometheus.io/port: "8002" + prometheus.io/scrape: "true" + labels: + app: contour + spec: + containers: + - args: + - -c + - /config/contour.yaml + - --service-cluster + - cluster0 + - --service-node + - node0 + command: + - envoy + image: docker.io/envoyproxy/envoy-alpine:v1.6.0 + name: envoy-envoyingressv1 + ports: + - containerPort: 80 + name: http + - containerPort: 8002 + name: statsd + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /config + name: contour-config + - args: + - serve + - --incluster + - --envoy-http-port=80 + - --debug-http-port=6069 + command: + - contour + image: gcr.io/heptio-images/contour:v0.6.1 + imagePullPolicy: Always + name: contour-unknown + ports: + - containerPort: 8000 + name: contour + dnsPolicy: ClusterFirst + initContainers: + - args: + - bootstrap + - /config/contour.yaml + - --statsd-enabled + command: + - contour + image: gcr.io/heptio-images/contour:v0.6.1 + imagePullPolicy: Always + name: envoy-initconfig + volumeMounts: + - mountPath: /config + name: contour-config + serviceAccountName: contour + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: contour-config +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/version: v2.4.0-v1alpha1 + name: sparkoperator + namespace: sparkoperator +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/version: v2.4.0-v1alpha1 + strategy: + type: Recreate + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + initializers: + pending: [] + labels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/version: v2.4.0-v1alpha1 + spec: + containers: + - args: + - -logtostderr + - -v=2 + - -controller-threads=20 + - -enable-metrics=true + - '-metrics-prefix=service:' + - -metrics-labels=task_name + - -metrics-labels=workflow_name + command: + - /usr/bin/spark-operator + image: gcr.io/spark-operator/spark-operator:v2.4.0-v1beta1-0.9.0 + imagePullPolicy: Always + name: sparkoperator-unknown + ports: + - containerPort: 10254 + serviceAccountName: sparkoperator +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: redis-resource-manager + serviceName: redis-resource-manager + template: + metadata: + labels: + app: redis-resource-manager + spec: + containers: + - env: + - name: REDIS_PASSWORD + value: mypassword + image: docker.io/bitnami/redis:4.0.2-r1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: redis-resource-manager + ports: + - containerPort: 6379 + name: redis + protocol: TCP + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 200m + memory: 128Mi + volumeMounts: + - mountPath: /bitnami + name: redis-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: syncresources + namespace: flyte +spec: + jobTemplate: + spec: + template: + spec: + containers: + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - clusterresource + - sync + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: sync-cluster-resources + volumeMounts: + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: OnFailure + volumes: + - configMap: + name: clusterresource-template-dgc9fcm2kh + name: resource-templates + - configMap: + name: flyte-admin-config-4gbbgmdf6d + name: config-volume + schedule: '*/1 * * * *' +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + name: flytesystem + namespace: flyte +spec: + rules: + - http: + paths: + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /console + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /__webpack_hmr + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /api + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /healthcheck + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /v1 + - backend: + serviceName: flyteadmin + servicePort: 81 + path: /flyteidl.service.AdminService + - backend: + serviceName: flyteadmin + servicePort: 87 + path: /openapi +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + name: minio + namespace: flyte +spec: + rules: + - http: + paths: + - backend: + serviceName: minio + servicePort: 9000 + path: /minio diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml new file mode 100644 index 0000000000..a1a4382d6a --- /dev/null +++ b/deployment/test/flyte_generated.yaml @@ -0,0 +1,1101 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flyte +spec: + finalizers: + - kubernetes +status: + phase: Active +--- +apiVersion: v1 +kind: Namespace +metadata: + name: heptio-contour +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: flyteworkflows.flyte.lyft.com +spec: + group: flyte.lyft.com + names: + kind: FlyteWorkflow + plural: flyteworkflows + shortNames: + - fly + singular: flyteworkflow + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + labels: + component: ingressroute + name: ingressroutes.contour.heptio.com +spec: + additionalPrinterColumns: + - JSONPath: .spec.virtualhost.fqdn + description: Fully qualified domain name + name: FQDN + type: string + - JSONPath: .spec.virtualhost.tls.secretName + description: Secret with TLS credentials + name: TLS Secret + type: string + - JSONPath: .spec.routes[0].match + description: First routes defined + name: First route + type: string + - JSONPath: .status.currentStatus + description: The current status of the IngressRoute + name: Status + type: string + - JSONPath: .status.description + description: Description of the current status + name: Status Description + type: string + group: contour.heptio.com + names: + kind: IngressRoute + plural: ingressroutes + scope: Namespaced + validation: + openAPIV3Schema: + properties: + spec: + properties: + healthCheck: + properties: + healthyThresholdCount: + type: integer + intervalSeconds: + type: integer + path: + pattern: ^\/.*$ + type: string + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + required: + - path + type: object + routes: + items: + properties: + delegate: + properties: + name: + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + namespace: + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + match: + pattern: ^\/.*$ + type: string + services: + items: + properties: + healthCheck: + properties: + healthyThresholdCount: + type: integer + intervalSeconds: + type: integer + path: + pattern: ^\/.*$ + type: string + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + required: + - path + type: object + name: + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + type: integer + strategy: + enum: + - RoundRobin + - WeightedLeastRequest + - Random + - RingHash + - Maglev + type: string + weight: + type: integer + required: + - name + - port + type: object + type: array + required: + - match + type: array + strategy: + enum: + - RoundRobin + - LeastRequest + - Random + - RingHash + - Maglev + type: string + virtualhost: + properties: + aliases: + items: + pattern: ^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$ + type: string + type: array + fqdn: + pattern: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-z]{2,}$ + type: string + required: + - routes + version: v1beta1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyteadmin + namespace: flyte +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flytepropeller + namespace: flyte +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: contour + namespace: heptio-contour +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyteadmin + namespace: flyte +rules: +- apiGroups: + - "" + - flyte.lyft.com + - rbac.authorization.k8s.io + resources: + - configmaps + - flyteworkflows + - namespaces + - pods + - roles + - rolebindings + - secrets + - services + - serviceaccounts + - spark-role + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flytepropeller +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - delete + - patch +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - flyte.lyft.com + resources: + - flyteworkflows + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - post + - deletecollection +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: contour +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - contour.heptio.com + resources: + - ingressroutes + verbs: + - get + - list + - watch + - put + - post + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flyteadmin-binding + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyteadmin +subjects: +- kind: ServiceAccount + name: flyteadmin + namespace: flyte +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flytepropeller + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flytepropeller +subjects: +- kind: ServiceAccount + name: flytepropeller + namespace: flyte +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: contour +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: contour +subjects: +- kind: ServiceAccount + name: contour + namespace: heptio-contour +--- +apiVersion: v1 +data: + aa_namespace.yaml: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes +kind: ConfigMap +metadata: + name: clusterresource-template-dgc9fcm2kh + namespace: flyte +--- +apiVersion: v1 +data: + flyteadmin_config.yaml: | + logger: + show-source: true + level: 5 + application: + httpPort: 8088 + grpcPort: 8089 + flyteadmin: + roleNameKey: "iam.amazonaws.com/role" + profilerPort: 10254 + metricsScope: "flyte:" + metadataStoragePrefix: + - "metadata" + - "admin" + testing: + host: http://flyteadmin + database: + port: 5432 + username: postgres + host: postgres + dbname: postgres + options: sslmode=disable + scheduler: + eventScheduler: + scheme: "local" + region: "us-east-1" + scheduleRole: "arn:aws:iam::abc123:role/my-schedule-role" + targetName: "arn:aws:sqs:my-region:abc123:my-queue" + workflowExecutor: + scheme: "local" + region: "us-east-1" + scheduleQueueName: "won't-work-locally" + accountId: "173840052742" + remoteData: + region: "us-east-1" + scheme: "local" + signedUrls: + durationMinutes: 3 + notifications: + type: local + region: "us-east-1" + publisher: + topicName: "foo" + processor: + queueName: "queue" + accountId: "bar" + queues: + executionQueues: + - parent: "FlyteNonProductionTaskCPU-staging-iad" + child: "FlyteNonProductionSubTaskCPU-staging-iad" + attributes: + - all + workflowConfigs: + - project: flytekit + tags: + - all + - project: "flytetester" + domain: "development" + workflowName: "app-workflows-rich-workflow-rich-workflow" + tags: + - all + - project: "flytetester" + domain: "development" + workflowName: "app-workflows-batch-workflow-batch-tasks-workflow" + tags: + - all + - tags: + - all + storage: + type: minio + connection: + access-key: minio + auth-type: accesskey + secret-key: miniostorage + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + container: "my-s3-bucket" + task_resources: + defaults: + cpu: 100m + memory: 100Mi + storage: 5Mi + limits: + cpu: 2 + memory: 2G + storage: 20Mi + task_type_whitelist: + spark: + - project: flytekit + - project: flytetester + domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + - id: domain + name: domain + cluster_resources: + templatePath: "/etc/flyte/clusterresource/templates" + refresh: 5m +kind: ConfigMap +metadata: + name: flyte-admin-config-4gbbgmdf6d + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + plugins: + enabled-plugins: + - container + - sidecar + logs: + kubernetes-enabled: true + kubernetes-url: "http://localhost:30082" + k8s: + default-env-vars: + - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" + - FLYTE_AWS_ACCESS_KEY_ID: minio + - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage +kind: ConfigMap +metadata: + name: flyte-plugin-config-g2248g9ftb + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + propeller: + metadata-prefix: metadata/propeller + workers: 4 + max-workflow-retries: 30 + workflow-reeval-duration: 30s + downstream-eval-duration: 30s + limit-namespace: "all" + prof-port: 10254 + metrics-prefix: flyte + enable-admin-launcher: true + leader-election: + lock-config-map: + name: propeller-leader + namespace: flyte + enabled: true + lease-duration: 15s + renew-deadline: 10s + retry-period: 2s + queue: + type: batch + batching-interval: 2s + batch-size: -1 + queue: + type: bucket + rate: 10 + capacity: 100 + sub-queue: + type: bucket + rate: 10 + capacity: 100 + logger: + show-source: true + level: 5 + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + type: minio + container: my-s3-bucket + event: + type: admin + rate: 500 + capacity: 1000 + admin: + endpoint: flyteadmin:81 + insecure: true +kind: ConfigMap +metadata: + name: flyte-propeller-config-d84t25m6f4 + namespace: flyte +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + contour.heptio.com/upstream-protocol.h2c: grpc + name: flyteadmin + namespace: flyte +spec: + ports: + - name: redoc + port: 87 + protocol: TCP + targetPort: 8087 + - name: http + port: 80 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 81 + protocol: TCP + targetPort: 8089 + selector: + app: flyteadmin +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: flyte +spec: + externalName: minio + ports: + - port: 9000 + selector: + app: minio +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: flyte +spec: + ports: + - port: 5432 + selector: + app: postgres +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: contour + name: contour + namespace: heptio-contour +spec: + ports: + - nodePort: 30081 + port: 80 + protocol: TCP + selector: + app: contour + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flyteadmin + name: flyteadmin + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: flyteadmin + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flyteadmin + spec: + containers: + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - serve + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: flyteadmin + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 200Mi + volumeMounts: + - mountPath: /srv/flyte + name: shared-data + - mountPath: /etc/flyte/config + name: config-volume + - command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: Flyte Admin OpenAPI + - name: SPEC_URL + value: /api/v1/openapi + - name: PORT + value: "8087" + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + name: redoc + ports: + - containerPort: 8087 + resources: + limits: + cpu: "0.1" + memory: 200Mi + initContainers: + - command: + - sh + - -c + - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep + 2; done; + image: postgres:10.1 + name: check-db-ready + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - migrate + - run + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: run-migrations + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - migrate + - seed-projects + - flytesnacks + - flytetester + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: seed-projects + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + serviceAccountName: flyteadmin + volumes: + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-admin-config-4gbbgmdf6d + name: config-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flytepropeller + name: flytepropeller + namespace: flyte +spec: + selector: + matchLabels: + app: flytepropeller + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flytepropeller + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/version: 0.1.9 + spec: + containers: + - args: + - --config + - /etc/flyte/config*/config.yaml + command: + - flytepropeller + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: docker.io/lyft/flytepropeller:v0.1.9 + imagePullPolicy: IfNotPresent + name: flytepropeller + ports: + - containerPort: 10254 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 100Mi + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + - mountPath: /etc/flyte/config-plugin + name: plugin-config-volume + serviceAccountName: flytepropeller + volumes: + - configMap: + name: flyte-propeller-config-d84t25m6f4 + name: config-volume + - configMap: + name: flyte-plugin-config-g2248g9ftb + name: plugin-config-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: flyte +spec: + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - args: + - server + - /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: miniostorage + image: minio/minio:RELEASE.2019-06-04T01-15-58Z + name: minio + ports: + - containerPort: 9000 + name: minio + volumeMounts: + - mountPath: /data + name: minio-storage + volumes: + - emptyDir: {} + name: minio-storage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: flyte +spec: + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - image: postgres:10.1 + name: postgres + ports: + - containerPort: 5432 + name: postgres + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgres-storage + volumes: + - emptyDir: {} + name: postgres-storage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: contour + name: contour + namespace: heptio-contour +spec: + replicas: 1 + selector: + matchLabels: + app: contour + template: + metadata: + annotations: + prometheus.io/format: prometheus + prometheus.io/path: /stats + prometheus.io/port: "8002" + prometheus.io/scrape: "true" + labels: + app: contour + spec: + containers: + - args: + - -c + - /config/contour.yaml + - --service-cluster + - cluster0 + - --service-node + - node0 + command: + - envoy + image: docker.io/envoyproxy/envoy-alpine:v1.6.0 + name: envoy-envoyingressv1 + ports: + - containerPort: 80 + name: http + - containerPort: 8002 + name: statsd + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /config + name: contour-config + - args: + - serve + - --incluster + - --envoy-http-port=80 + - --debug-http-port=6069 + command: + - contour + image: gcr.io/heptio-images/contour:v0.6.1 + imagePullPolicy: Always + name: contour-unknown + ports: + - containerPort: 8000 + name: contour + dnsPolicy: ClusterFirst + initContainers: + - args: + - bootstrap + - /config/contour.yaml + - --statsd-enabled + command: + - contour + image: gcr.io/heptio-images/contour:v0.6.1 + imagePullPolicy: Always + name: envoy-initconfig + volumeMounts: + - mountPath: /config + name: contour-config + serviceAccountName: contour + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: contour-config +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: redis-resource-manager + serviceName: redis-resource-manager + template: + metadata: + labels: + app: redis-resource-manager + spec: + containers: + - env: + - name: REDIS_PASSWORD + value: mypassword + image: docker.io/bitnami/redis:4.0.2-r1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: redis-resource-manager + ports: + - containerPort: 6379 + name: redis + protocol: TCP + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 200m + memory: 128Mi + volumeMounts: + - mountPath: /bitnami + name: redis-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: syncresources + namespace: flyte +spec: + jobTemplate: + spec: + template: + spec: + containers: + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - clusterresource + - sync + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: sync-cluster-resources + volumeMounts: + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: OnFailure + volumes: + - configMap: + name: clusterresource-template-dgc9fcm2kh + name: resource-templates + - configMap: + name: flyte-admin-config-4gbbgmdf6d + name: config-volume + schedule: '*/1 * * * *' +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + name: flytesystem + namespace: flyte +spec: + rules: + - http: + paths: + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /console + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /__webpack_hmr + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /api + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /healthcheck + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /v1 + - backend: + serviceName: flyteadmin + servicePort: 81 + path: /flyteidl.service.AdminService + - backend: + serviceName: flyteadmin + servicePort: 87 + path: /openapi +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + name: minio + namespace: flyte +spec: + rules: + - http: + paths: + - backend: + serviceName: minio + servicePort: 9000 + path: /minio diff --git a/docs_infra/in_container_html_generation.sh b/docs_infra/in_container_html_generation.sh new file mode 100755 index 0000000000..2743645b0f --- /dev/null +++ b/docs_infra/in_container_html_generation.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# This file is meant to be run within the docker container, and mounted +# /rsts as the inputs dir +# /docs as the output dir +# /base as the dir where this script lives +# docker run -t -v ${BASEDIR}:/base -v ${BASEDIR}/docs:/docs -v ${BASEDIR}/rsts_tmp:/rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_html_generation.sh +# +# It will expect flyteidl and flytekit checked out at the /repos folder. + +set -e +set -x + +# We're generating documentation for flytekit, which references flyteidl. autodoc complains if the python libraries +# referenced are not installed, so let's install them +pip install -U flytekit[all] + +sphinx-build -Q -b html -c /rsts /rsts /docs/ diff --git a/docs_infra/in_container_rst_generation.sh b/docs_infra/in_container_rst_generation.sh new file mode 100755 index 0000000000..54f8ed3cf5 --- /dev/null +++ b/docs_infra/in_container_rst_generation.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This file is meant to be run within the docker container that has the appropriate repos checked out, and mounted +# docker run -t -v ${BASEDIR}:/base -v ${REPOS_DIR}:/repos -v ${BASEDIR}/_rsts:/_rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_rst_generation.sh +# +# It will expect flyteidl and flytekit checked out at the /repos folder. + +set -e +set -x + +# We're generating documentation for flytesdk, which references flyteidl. autodoc complains if the python libraries +# referenced are not installed, so let's install them +pip install -U flytekit[all] + +# Generate the RST files for flytekit +sphinx-apidoc --force --tocfile index --ext-autodoc --output-dir /repos/flytekit/docs /repos/flytekit/flytekit + +# Create a temp directory where we can aggregate all the rst files from all locations +mkdir /rsts + +# The toctree in this index file requires that the idl/sdk rsts are in the same folder +cp -R /repos/flyteidl/gen/pb-protodoc/flyteidl /rsts/ +cp -R /repos/flytekit/docs /rsts/flytekit + +cp -R /rsts/* /_rsts diff --git a/docs_infra/index.rst b/docs_infra/index.rst new file mode 100644 index 0000000000..1c5a807052 --- /dev/null +++ b/docs_infra/index.rst @@ -0,0 +1,22 @@ +Flyte +===== + +About Flyte + +.. toctree:: + :maxdepth: 2 + :caption: Flyte Documentation + :name: mastertoc + + introduction/index + user/index + administrator/index + contributor/index + +.. toctree:: + :caption: Generated Documentation from Source + :maxdepth: 1 + :name: sourcecodedocstoc + + flyteidl/index + Flytekit diff --git a/end2end/Readme.rst b/end2end/Readme.rst new file mode 100644 index 0000000000..bd0c0fac14 --- /dev/null +++ b/end2end/Readme.rst @@ -0,0 +1,26 @@ +Flyte End-to-End Testing +=============================== + +Flyte has implemented some light constructs around testing the platform end-to-end. Service level integration tests generally only deal with API boundaries across one layer. Service A might talk to Service B and to Service A's database in an integration test. End-to-end testing is meant to simulate the entire Flyte platform, from a user's workflow code, all the way from registration to execution and verification of results. + +Design +########## + +Infrastructure +**************** +End-to-end testing uses the `Dockernetes `_ container that the Flyte team has put together. This is a Docker container, that runs a Kubernetes cluster within it. That means that from the Docker container, you have access to ``kubectl``. Dockernetes doesn't currently give you access to the images that the Docker daemon running Dockernetes already has, but this is a feature that we may one day build out. + + +Container Entrypoint +********************** +Any workflow repository that will be the target of an end-to-end test will need to have the ``end2end_test`` Make target available. This is the common entrypoint that the tooling in this repository will call. + +Background +*********** +End-to-end testing was initally going to be implemented with K8s Jobs. We thought that we'd have somewhere on the order of 100 jobs. In implementing tests, we quickly realized that managing that number of jobs would be a lot of work - keeping track of timeouts, failures, partial pod failures like ImagePullBackoff issues, retries, etc. What we'd end up writing for that would look very much like a higher level Jobs controller. That would be way too much work. + +We thought about directly running Docker, like ``docker run flytetester make endtoend``. That would work, except that we still have some networking limitations around gRPC ingress at time of writing, and port-forwarding within the Dockernetes container sounded like too much virtualization hocus-pocus. + +So we stuck with the in-cluster idea, but decided to just keep it simple - run a Pod instead of a Job, and use simple Bash constructs to tell when things are done. And keep the number of tests small, like one or two, at least for now. + +For future tests, it may be best to run different execute.sh scripts. That is, instead of multiple end-to-end tests in one Dockernetes container/cluster, each end-to-end test gets its own Dockernetes container/cluster. This keeps things simpler, and also enables tests to do things like write a workflow with one version of FlyteAdmin, then upgrade Admin, and read it back with a newer version. diff --git a/end2end/dockernetes_run.sh b/end2end/dockernetes_run.sh new file mode 100755 index 0000000000..67de1f569b --- /dev/null +++ b/end2end/dockernetes_run.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -e + +echo "" +echo "waiting up to 5 minutes for kubernetes to start..." + +K8S_TIMEOUT="300" + +SECONDS=0 +while ! systemctl is-active --quiet multi-user.target; do + sleep 2 + if [ "$SECONDS" -gt "$K8S_TIMEOUT" ]; then + echo "ERROR: timed out waiting for kubernetes to start." + exit 1 + fi +done + +echo "kubernetes started in $SECONDS seconds." +echo "" + +exec $1 diff --git a/end2end/execute.sh b/end2end/execute.sh new file mode 100755 index 0000000000..a0aa99806a --- /dev/null +++ b/end2end/execute.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Fail if any command fails and print what this script is doing +set -ex +# Also fail in the middle of a pipe if we ever have pipes +set -o pipefail +# Apply traps to all function calls +set -o errtrace + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +# Dump logs out because trying to upload to Travis artifacts is too hard. +# This is the naive solution, think of a better one in the future. +trap $DIR/print_logs.sh ERR + +# launch flyte end2end +kubectl apply -f "$DIR/../deployment/test/flyte_generated.yaml" + +# in debug mode, run bash instead of running the tests +if [ -n "$DOCKERNETES_DEBUG" ]; then + bash +fi + +# This is a separate function so that we can potentially reuse in the future when we have more than one test +function wait_for_flyte_deploys() { + SECONDS=0 + echo "" + echo "waiting for flyte deploy to complete..." + echo "" + + # wait for flyteadmin deployment to complete + kubectl -n flyte rollout status deployment flyteadmin + echo "" + kubectl -n flyte rollout status deployment flytepropeller + echo "" + kubectl -n heptio-contour rollout status deployment contour + echo "" + + echo "Flyte deployed in $SECONDS seconds." +} + +function run_flyte_examples() +{ + echo $DIR + # Launch test + kubectl -n flyte create -f $DIR/tests/endtoend.yaml + # Wait at most five minutes for things to pass + /usr/bin/timeout 600 $DIR/test_monitor.sh + return $? +} + +wait_for_flyte_deploys + +# In the future, to run additional tests against different containers, you can delete/reapply the K8s file +# to reset the cluster, wait for deployments again, and run additional functions. +run_flyte_examples +# Dump the logs from the run, automatically happens on error because of the trap, but we want this +# even for successful runs +# TODO: Move into the above function in the future +kubectl -n flyte logs endtoend + diff --git a/end2end/launch_dockernetes.sh b/end2end/launch_dockernetes.sh new file mode 100755 index 0000000000..7b601c8479 --- /dev/null +++ b/end2end/launch_dockernetes.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +# stop any existing test container that might be running +docker kill dockernetes || true + +# initialize the k8s container, mounting the flyte repo to /flyte. + +# The container must start with systemd (/sbin/init) as PID 1 + +docker run \ + --detach \ + --rm \ + --privileged \ + --volume /var/lib/docker \ + --volume /lib/modules:/lib/modules \ + --volume $DIR/..:/flyte \ + --name dockernetes \ + --env "DOCKERNETES_DEBUG=${DOCKERNETES_DEBUG}" \ + lyft/dockernetes:7692164d7e6b3963bbcc39a3f5510495509cb71a /sbin/init + +# wait for the system to initalize, then run execute.sh +docker exec \ + -it \ + dockernetes /flyte/end2end/dockernetes_run.sh /flyte/end2end/execute.sh diff --git a/end2end/print_logs.sh b/end2end/print_logs.sh new file mode 100755 index 0000000000..c97bbe4a85 --- /dev/null +++ b/end2end/print_logs.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -ex + +function get_flyte_pods() { + echo $(kubectl -n flyte get pods | awk '{print $1}' | grep -v NAME) +} + +pods=$(get_flyte_pods) +echo $pods | while read -a podarray; do + for i in "${podarray[@]}"; do + echo "Logs for ${i}" + echo "------------------------------------------" + kubectl -n flyte logs $i + done +done + diff --git a/end2end/test_monitor.sh b/end2end/test_monitor.sh new file mode 100755 index 0000000000..21be0b2abe --- /dev/null +++ b/end2end/test_monitor.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -ex +set -o pipefail + +function pod_succeeded() +{ + kubectl -n flyte get pod endtoend -o=custom-columns=STATUS:.status.phase | grep Succeeded > /dev/null + return $? +} + +function get_status() +{ + kubectl -n flyte get pod endtoend -o=custom-columns=STATUS:.status.phase | grep -v STATUS +} + +function wait_until_terminal_state() +{ + status=$(get_status) + while [[ "$status" != "Succeeded" && "$status" != "Failed" ]]; do + echo "Status is still $status" + sleep 5 + status=$(get_status) + done +} + +function watch_pod() +{ + wait_until_terminal_state + return $(pod_succeeded) +} + +watch_pod + diff --git a/end2end/tests/endtoend.yaml b/end2end/tests/endtoend.yaml new file mode 100644 index 0000000000..6d777f5d63 --- /dev/null +++ b/end2end/tests/endtoend.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: endtoend + namespace: flyte +spec: + containers: + - args: + - make end2end_test + command: + - bash + - -c + image: docker.io/lyft/flytetester:v0.1.0 + imagePullPolicy: IfNotPresent + name: flytetester + resources: + requests: + cpu: "1" + memory: 500Mi + dnsPolicy: ClusterFirst + priority: 0 + restartPolicy: Never diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml new file mode 100644 index 0000000000..7187f483cf --- /dev/null +++ b/kustomize/base/admindeployment/deployment.yaml @@ -0,0 +1,70 @@ +# Create the actual deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteadmin + namespace: flyte + labels: + app: flyteadmin +spec: + replicas: 1 + selector: + matchLabels: + app: flyteadmin + template: + metadata: + labels: + app: flyteadmin + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: flyteadmin + volumes: + - name: shared-data + emptyDir: {} + - name: config-volume + configMap: + name: flyte-admin-config + initContainers: + - name: run-migrations + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + containers: + - name: flyteadmin + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "serve"] + ports: + - containerPort: 8088 + - containerPort: 8089 + volumeMounts: + - name: shared-data + mountPath: /srv/flyte + - name: config-volume + mountPath: /etc/flyte/config + - name: redoc + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8087 + resources: + limits: + memory: "200Mi" + cpu: "0.1" + command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: "Flyte Admin OpenAPI" + - name: SPEC_URL + value: "/api/v1/openapi" + - name: PORT + value: "8087" diff --git a/kustomize/base/admindeployment/kustomization.yaml b/kustomize/base/admindeployment/kustomization.yaml new file mode 100644 index 0000000000..a944d005ca --- /dev/null +++ b/kustomize/base/admindeployment/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- deployment.yaml +- service.yaml diff --git a/kustomize/base/admindeployment/service.yaml b/kustomize/base/admindeployment/service.yaml new file mode 100644 index 0000000000..18f80a3763 --- /dev/null +++ b/kustomize/base/admindeployment/service.yaml @@ -0,0 +1,25 @@ +# Service +apiVersion: v1 +kind: Service +metadata: + name: flyteadmin + namespace: flyte + annotations: + # This tells contour to use a H2 connection for the port associated + # with the name 'grpc' under spec/ports. + # For more information, refer to + # https://github.com/heptio/contour/blob/master/docs/annotations.md#contour-specific-service-annotations + contour.heptio.com/upstream-protocol.h2c: "grpc" +spec: + selector: + app: flyteadmin + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 8088 + - name: grpc + protocol: TCP + port: 81 + targetPort: 8089 + diff --git a/kustomize/base/adminserviceaccount/adminserviceaccount.yaml b/kustomize/base/adminserviceaccount/adminserviceaccount.yaml new file mode 100644 index 0000000000..a4c7c03755 --- /dev/null +++ b/kustomize/base/adminserviceaccount/adminserviceaccount.yaml @@ -0,0 +1,49 @@ +# ClusterRole for flyteadmin +# https://kubernetes.io/docs/admin/authorization/rbac/ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flyteadmin + namespace: flyte +rules: +#Allow Access to all resources under flyte.lyft.com and the core API group: "" + - apiGroups: + - "" + - flyte.lyft.com + - rbac.authorization.k8s.io + resources: + - configmaps + - flyteworkflows + - namespaces + - pods + - roles + - rolebindings + - secrets + - services + - serviceaccounts + - spark-role + verbs: + - "*" + +--- +# Create a Service Account for FltyeAdmin +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyteadmin + namespace: flyte + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flyteadmin-binding + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyteadmin +subjects: + - kind: ServiceAccount + name: flyteadmin + namespace: flyte diff --git a/kustomize/base/adminserviceaccount/kustomization.yaml b/kustomize/base/adminserviceaccount/kustomization.yaml new file mode 100644 index 0000000000..3d4938e1a1 --- /dev/null +++ b/kustomize/base/adminserviceaccount/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- adminserviceaccount.yaml diff --git a/kustomize/base/console/configmap.yaml b/kustomize/base/console/configmap.yaml new file mode 100644 index 0000000000..0fb42a0c1f --- /dev/null +++ b/kustomize/base/console/configmap.yaml @@ -0,0 +1,10 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: flyte-console-config + namespace: flyte +data: + # This determines the base url used for all console links. It must match the route specified in ingress.yaml + BASE_URL: /console + CONFIG_DIR: /etc/flyte/config + diff --git a/kustomize/base/console/deployment.yaml b/kustomize/base/console/deployment.yaml new file mode 100644 index 0000000000..ffcfcca44e --- /dev/null +++ b/kustomize/base/console/deployment.yaml @@ -0,0 +1,33 @@ +# Create the actual deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteconsole + namespace: flyte + labels: + app: flyteconsole +spec: + replicas: 1 + selector: + matchLabels: + app: flyteconsole + template: + metadata: + labels: + app: flyteconsole + spec: + volumes: + - name: shared-data + emptyDir: {} + containers: + - name: flyteconsole + image: docker.io/lyft/flyteconsole:v0.1.0 + # args: [] + ports: + - containerPort: 8080 + volumeMounts: + - name: shared-data + mountPath: /srv/flyte + envFrom: + - configMapRef: + name: flyte-console-config diff --git a/kustomize/base/console/kustomization.yaml b/kustomize/base/console/kustomization.yaml new file mode 100644 index 0000000000..efc0d46a0b --- /dev/null +++ b/kustomize/base/console/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- deployment.yaml +- configmap.yaml +- service.yaml diff --git a/kustomize/base/console/service.yaml b/kustomize/base/console/service.yaml new file mode 100644 index 0000000000..e6d75ffc2f --- /dev/null +++ b/kustomize/base/console/service.yaml @@ -0,0 +1,14 @@ +# Service +apiVersion: v1 +kind: Service +metadata: + name: flyteconsole + namespace: flyte +spec: + selector: + app: flyteconsole + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + diff --git a/kustomize/base/datacatalog/deployment.yaml b/kustomize/base/datacatalog/deployment.yaml new file mode 100644 index 0000000000..af276e8340 --- /dev/null +++ b/kustomize/base/datacatalog/deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datacatalog + namespace: flyte + labels: + app: datacatalog +spec: + replicas: 1 + selector: + matchLabels: + app: datacatalog + template: + metadata: + labels: + app: datacatalog + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: datacatalog + volumes: + - name: shared-data + emptyDir: {} + - name: config-volume + configMap: + name: datacatalog-config + initContainers: + - name: run-migrations + image: docker.io/lyft/datacatalog:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["datacatalog", "--logtostderr", "--config", "/etc/datacatalog/config/datacatalog_config.yaml", "migrate", "run"] + volumeMounts: + - name: config-volume + mountPath: /etc/datacatalog/config + containers: + - name: datacatalog + image: docker.io/lyft/datacatalog:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["datacatalog", "--logtostderr", "--config", "/etc/datacatalog/config/datacatalog_config.yaml", "serve"] + ports: + - containerPort: 8088 + - containerPort: 8089 + volumeMounts: + - name: config-volume + mountPath: /etc/datacatalog/config diff --git a/kustomize/base/datacatalog/kustomization.yaml b/kustomize/base/datacatalog/kustomization.yaml new file mode 100644 index 0000000000..a944d005ca --- /dev/null +++ b/kustomize/base/datacatalog/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- deployment.yaml +- service.yaml diff --git a/kustomize/base/datacatalog/service.yaml b/kustomize/base/datacatalog/service.yaml new file mode 100644 index 0000000000..11c9b58ea2 --- /dev/null +++ b/kustomize/base/datacatalog/service.yaml @@ -0,0 +1,24 @@ +# Service +apiVersion: v1 +kind: Service +metadata: + name: datacatalog + namespace: flyte + annotations: + # This tells contour to use a H2 connection for the port associated + # with the name 'grpc' under spec/ports. + # For more information, refer to + # https://github.com/heptio/contour/blob/master/docs/annotations.md#contour-specific-service-annotations + contour.heptio.com/upstream-protocol.h2c: "grpc" +spec: + selector: + app: datacatalog + ports: + - name: http + protocol: TCP + port: 88 + targetPort: 8088 + - name: grpc + protocol: TCP + port: 89 + targetPort: 8089 diff --git a/kustomize/base/ingress/ingress.yaml b/kustomize/base/ingress/ingress.yaml new file mode 100644 index 0000000000..431cbede0a --- /dev/null +++ b/kustomize/base/ingress/ingress.yaml @@ -0,0 +1,44 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: flytesystem + namespace: flyte + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" +spec: + rules: + - http: + paths: + # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml + - path: /console + backend: + serviceName: flyteconsole + servicePort: 80 + # This is useful only for sandbox mode and should be templatized/removed in non-sandbox environments + - path: /__webpack_hmr + backend: + serviceName: flyteconsole + servicePort: 80 + - path: /api + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /healthcheck + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /v1 + backend: + serviceName: flyteadmin + servicePort: 80 + # NOTE: Port 81 in flyteadmin is the GRPC server port for + # FlyteAdmin. + - path: /flyteidl.service.AdminService + backend: + serviceName: flyteadmin + servicePort: 81 + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi + backend: + serviceName: flyteadmin + servicePort: 87 diff --git a/kustomize/base/ingress/kustomization.yaml b/kustomize/base/ingress/kustomization.yaml new file mode 100644 index 0000000000..36e8c40ae5 --- /dev/null +++ b/kustomize/base/ingress/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- ingress.yaml diff --git a/kustomize/base/namespace/kustomization.yaml b/kustomize/base/namespace/kustomization.yaml new file mode 100644 index 0000000000..bf20f4df68 --- /dev/null +++ b/kustomize/base/namespace/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- namespace.yaml diff --git a/kustomize/base/namespace/namespace.yaml b/kustomize/base/namespace/namespace.yaml new file mode 100644 index 0000000000..a5328890f7 --- /dev/null +++ b/kustomize/base/namespace/namespace.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flyte +spec: + finalizers: + - kubernetes +status: + phase: Active +--- diff --git a/kustomize/base/operators/spark/deployment.yaml b/kustomize/base/operators/spark/deployment.yaml new file mode 100644 index 0000000000..39558d3ada --- /dev/null +++ b/kustomize/base/operators/spark/deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: sparkoperator + namespace: sparkoperator + labels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/version: v2.4.0-v1alpha1 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/version: v2.4.0-v1alpha1 + strategy: + type: Recreate + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" + prometheus.io/path: "/metrics" + labels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/version: v2.4.0-v1alpha1 + initializers: + pending: [] + spec: + serviceAccountName: sparkoperator + containers: + - name: sparkoperator-unknown + image: gcr.io/spark-operator/spark-operator:v2.4.0-v1beta1-0.9.0 + imagePullPolicy: Always + command: ["/usr/bin/spark-operator"] + ports: + - containerPort: 10254 + args: + - -logtostderr + - -v=2 + - -controller-threads=20 + - -enable-metrics=true + - "-metrics-prefix=service:" + - -metrics-labels=task_name + - -metrics-labels=workflow_name diff --git a/kustomize/base/operators/spark/kustomization.yaml b/kustomize/base/operators/spark/kustomization.yaml new file mode 100644 index 0000000000..39772f0480 --- /dev/null +++ b/kustomize/base/operators/spark/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- spark-operator.yaml +- spark-dummy-crd.yaml +- deployment.yaml diff --git a/kustomize/base/operators/spark/spark-dummy-crd.yaml b/kustomize/base/operators/spark/spark-dummy-crd.yaml new file mode 100644 index 0000000000..2357755ef7 --- /dev/null +++ b/kustomize/base/operators/spark/spark-dummy-crd.yaml @@ -0,0 +1,15 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: sparkapplications.sparkoperator.k8s.io +spec: + group: sparkoperator.k8s.io + names: + kind: SparkApplication + listKind: SparkApplicationList + plural: sparkapplications + shortNames: + - sparkapp + singular: sparkapplication + scope: Namespaced + version: v1beta1 \ No newline at end of file diff --git a/kustomize/base/operators/spark/spark-operator.yaml b/kustomize/base/operators/spark/spark-operator.yaml new file mode 100644 index 0000000000..7e1eabc4d7 --- /dev/null +++ b/kustomize/base/operators/spark/spark-operator.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sparkoperator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sparkoperator + namespace: sparkoperator +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: sparkoperator +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["*"] +- apiGroups: [""] + resources: ["services", "configmaps"] + verbs: ["create", "get", "delete"] +- apiGroups: ["extensions"] + resources: ["ingresses"] + verbs: ["create", "get", "delete"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "update", "delete", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "update", "delete"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: ["create", "get", "update", "delete"] +- apiGroups: ["sparkoperator.k8s.io"] + resources: ["sparkapplications", "scheduledsparkapplications"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: sparkoperator +subjects: + - kind: ServiceAccount + name: sparkoperator + namespace: sparkoperator +roleRef: + kind: ClusterRole + name: sparkoperator + apiGroup: rbac.authorization.k8s.io diff --git a/kustomize/base/propeller/deployment.yaml b/kustomize/base/propeller/deployment.yaml new file mode 100644 index 0000000000..cd4a488381 --- /dev/null +++ b/kustomize/base/propeller/deployment.yaml @@ -0,0 +1,52 @@ +# Create the actual deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte + labels: + app: flytepropeller +spec: + selector: + matchLabels: + app: flytepropeller + template: + metadata: + labels: + app: flytepropeller + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/version: 0.1.9 + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: flytepropeller + volumes: + - name: config-volume + configMap: + name: flyte-propeller-config + - name: plugin-config-volume + configMap: + name: flyte-plugin-config + containers: + - name: flytepropeller + image: docker.io/lyft/flytepropeller:v0.1.9 + command: + - flytepropeller + args: + - --config + - /etc/flyte/config*/config.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + imagePullPolicy: IfNotPresent + ports: + - containerPort: 10254 + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: plugin-config-volume + mountPath: /etc/flyte/config-plugin diff --git a/kustomize/base/propeller/kustomization.yaml b/kustomize/base/propeller/kustomization.yaml new file mode 100644 index 0000000000..3f090bdf48 --- /dev/null +++ b/kustomize/base/propeller/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- deployment.yaml +- rbac.yaml diff --git a/kustomize/base/propeller/rbac.yaml b/kustomize/base/propeller/rbac.yaml new file mode 100644 index 0000000000..524740005e --- /dev/null +++ b/kustomize/base/propeller/rbac.yaml @@ -0,0 +1,88 @@ +# Create a ClusterRole for flytepropeller +# https://kubernetes.io/docs/admin/authorization/rbac/ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flytepropeller +rules: +#Allow RO access to PODS + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +#Allow Event recording access + - apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - delete + - patch +#Allow Access All plugin objects + - apiGroups: + - "*" + resources: + - "*" + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +#Allow Access to CRD + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete + - update +#Allow Access to all resources under flyte.lyft.com + - apiGroups: + - flyte.lyft.com + resources: + - flyteworkflows + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - post + - deletecollection +--- +# Create a Service Account for Flytepropeller +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flytepropeller + namespace: flyte +--- +# Create a binding from Role -> ServiceAccount +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flytepropeller + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flytepropeller +subjects: +- kind: ServiceAccount + name: flytepropeller + namespace: flyte diff --git a/kustomize/base/wf_crd/kustomization.yaml b/kustomize/base/wf_crd/kustomization.yaml new file mode 100644 index 0000000000..6f680eaac2 --- /dev/null +++ b/kustomize/base/wf_crd/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- wf_crd.yaml diff --git a/kustomize/base/wf_crd/wf_crd.yaml b/kustomize/base/wf_crd/wf_crd.yaml new file mode 100644 index 0000000000..77eea97bd1 --- /dev/null +++ b/kustomize/base/wf_crd/wf_crd.yaml @@ -0,0 +1,22 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + # name must match the spec fields below, and be in the form: . + name: flyteworkflows.flyte.lyft.com +spec: + # group name to use for REST API: /apis// + group: flyte.lyft.com + # version name to use for REST API: /apis// + version: v1alpha1 + # either Namespaced or Cluster + names: + # plural name to be used in the URL: /apis/// + plural: flyteworkflows + # singular name to be used as an alias on the CLI and for display + singular: flyteworkflow + # kind is normally the CamelCased singular type. Your resource manifests use this. + kind: FlyteWorkflow + # shortNames allow shorter string to match your resource on the CLI + shortNames: + - fly + scope: Namespaced diff --git a/kustomize/dependencies/contour_ingress_controller/custom-config.yaml b/kustomize/dependencies/contour_ingress_controller/custom-config.yaml new file mode 100644 index 0000000000..c0fcfee56a --- /dev/null +++ b/kustomize/dependencies/contour_ingress_controller/custom-config.yaml @@ -0,0 +1,134 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressroutes.contour.heptio.com + labels: + component: ingressroute +spec: + group: contour.heptio.com + version: v1beta1 + scope: Namespaced + names: + plural: ingressroutes + kind: IngressRoute + additionalPrinterColumns: + - name: FQDN + type: string + description: Fully qualified domain name + JSONPath: .spec.virtualhost.fqdn + - name: TLS Secret + type: string + description: Secret with TLS credentials + JSONPath: .spec.virtualhost.tls.secretName + - name: First route + type: string + description: First routes defined + JSONPath: .spec.routes[0].match + - name: Status + type: string + description: The current status of the IngressRoute + JSONPath: .status.currentStatus + - name: Status Description + type: string + description: Description of the current status + JSONPath: .status.description + validation: + openAPIV3Schema: + properties: + spec: + required: + - routes + properties: + virtualhost: + properties: + fqdn: + type: string + pattern: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-z]{2,}$ + aliases: + type: array + items: + type: string + pattern: ^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$ + strategy: + type: string + enum: + - RoundRobin + - LeastRequest + - Random + - RingHash + - Maglev + healthCheck: + type: object + required: + - path + properties: + path: + type: string + pattern: ^\/.*$ + intervalSeconds: + type: integer + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + healthyThresholdCount: + type: integer + routes: + type: array + items: + required: + - match + properties: + match: + type: string + pattern: ^\/.*$ + delegate: + type: object + required: + - name + properties: + name: + type: string + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 + namespace: + type: string + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 + services: + type: array + items: + type: object + required: + - name + - port + properties: + name: + type: string + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 + port: + type: integer + weight: + type: integer + strategy: + type: string + enum: + - RoundRobin + - WeightedLeastRequest + - Random + - RingHash + - Maglev + healthCheck: + type: object + required: + - path + properties: + path: + type: string + pattern: ^\/.*$ + intervalSeconds: + type: integer + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + healthyThresholdCount: + type: integer diff --git a/kustomize/dependencies/contour_ingress_controller/deployment.yaml b/kustomize/dependencies/contour_ingress_controller/deployment.yaml new file mode 100644 index 0000000000..c24f2b504c --- /dev/null +++ b/kustomize/dependencies/contour_ingress_controller/deployment.yaml @@ -0,0 +1,66 @@ +# Create the actual deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: contour + namespace: heptio-contour + labels: + app: contour +spec: + replicas: 1 + selector: + matchLabels: + app: contour + template: + metadata: + labels: + app: contour + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8002" + prometheus.io/path: "/stats" + prometheus.io/format: "prometheus" + spec: + containers: + - image: docker.io/envoyproxy/envoy-alpine:v1.6.0 + name: envoy-envoyingressv1 + ports: + - containerPort: 80 + name: http + - containerPort: 8002 + name: statsd + command: ["envoy"] + args: ["-c", "/config/contour.yaml", "--service-cluster", "cluster0", "--service-node", "node0"] + volumeMounts: + - name: contour-config + mountPath: /config + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 100m + memory: 100Mi + - image: gcr.io/heptio-images/contour:v0.6.1 + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: contour + name: contour-unknown + command: ["contour"] + args: ["serve", "--incluster", "--envoy-http-port=80", "--debug-http-port=6069"] + initContainers: + - image: gcr.io/heptio-images/contour:v0.6.1 + imagePullPolicy: Always + name: envoy-initconfig + command: ["contour"] + args: ["bootstrap", "/config/contour.yaml", "--statsd-enabled"] + volumeMounts: + - name: contour-config + mountPath: /config + volumes: + - name: contour-config + emptyDir: {} + dnsPolicy: ClusterFirst + serviceAccountName: contour + terminationGracePeriodSeconds: 30 diff --git a/kustomize/dependencies/contour_ingress_controller/kustomization.yaml b/kustomize/dependencies/contour_ingress_controller/kustomization.yaml new file mode 100644 index 0000000000..36ba42801d --- /dev/null +++ b/kustomize/dependencies/contour_ingress_controller/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- namespace.yaml +- rbac.yaml +- custom-config.yaml +- service.yaml +- deployment.yaml diff --git a/kustomize/dependencies/contour_ingress_controller/namespace.yaml b/kustomize/dependencies/contour_ingress_controller/namespace.yaml new file mode 100644 index 0000000000..4be554a003 --- /dev/null +++ b/kustomize/dependencies/contour_ingress_controller/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: heptio-contour diff --git a/kustomize/dependencies/contour_ingress_controller/rbac.yaml b/kustomize/dependencies/contour_ingress_controller/rbac.yaml new file mode 100644 index 0000000000..4492a6f80f --- /dev/null +++ b/kustomize/dependencies/contour_ingress_controller/rbac.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: contour + namespace: heptio-contour +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: contour +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: ["contour.heptio.com"] + resources: ["ingressroutes"] + verbs: + - get + - list + - watch + - put + - post + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: contour +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: contour +subjects: +- kind: ServiceAccount + name: contour + namespace: heptio-contour diff --git a/kustomize/dependencies/contour_ingress_controller/service.yaml b/kustomize/dependencies/contour_ingress_controller/service.yaml new file mode 100644 index 0000000000..b7e10ab4af --- /dev/null +++ b/kustomize/dependencies/contour_ingress_controller/service.yaml @@ -0,0 +1,17 @@ +# Service +apiVersion: v1 +kind: Service +metadata: + name: contour + namespace: heptio-contour + labels: + app: contour +spec: + # use NodePort to make sure the service is accessible + type: NodePort + selector: + app: contour + ports: + - protocol: TCP + port: 80 + nodePort: 30081 diff --git a/kustomize/dependencies/database/database.yaml b/kustomize/dependencies/database/database.yaml new file mode 100644 index 0000000000..b85eea088a --- /dev/null +++ b/kustomize/dependencies/database/database.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: flyte +spec: + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + volumes: + - name: postgres-storage + emptyDir: {} + containers: + - image: "postgres:10.1" + name: postgres + ports: + - containerPort: 5432 + name: postgres + volumeMounts: + - name: postgres-storage + mountPath: /var/lib/postgresql/data +--- + apiVersion: v1 + kind: Service + metadata: + name: postgres + namespace: flyte + spec: + ports: + - port: 5432 + selector: + app: postgres diff --git a/kustomize/dependencies/database/kustomization.yaml b/kustomize/dependencies/database/kustomization.yaml new file mode 100644 index 0000000000..ca2ede8d46 --- /dev/null +++ b/kustomize/dependencies/database/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- database.yaml diff --git a/kustomize/dependencies/redis/deployment.yaml b/kustomize/dependencies/redis/deployment.yaml new file mode 100644 index 0000000000..6d02c0faac --- /dev/null +++ b/kustomize/dependencies/redis/deployment.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: redis-resource-manager + serviceName: redis-resource-manager + template: + metadata: + labels: + app: redis-resource-manager + spec: + containers: + - env: + - name: REDIS_PASSWORD + value: mypassword + image: docker.io/bitnami/redis:4.0.2-r1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: redis-resource-manager + ports: + - containerPort: 6379 + name: redis + protocol: TCP + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 200m + memory: 128Mi + volumeMounts: + - mountPath: /bitnami + name: redis-data + dnsPolicy: ClusterFirst + restartPolicy: Always diff --git a/kustomize/dependencies/redis/kustomization.yaml b/kustomize/dependencies/redis/kustomization.yaml new file mode 100644 index 0000000000..a944d005ca --- /dev/null +++ b/kustomize/dependencies/redis/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- deployment.yaml +- service.yaml diff --git a/kustomize/dependencies/redis/service.yaml b/kustomize/dependencies/redis/service.yaml new file mode 100644 index 0000000000..8c86264ae3 --- /dev/null +++ b/kustomize/dependencies/redis/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP diff --git a/kustomize/dependencies/storage/kustomization.yaml b/kustomize/dependencies/storage/kustomization.yaml new file mode 100644 index 0000000000..9a80610c4e --- /dev/null +++ b/kustomize/dependencies/storage/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- storage.yaml diff --git a/kustomize/dependencies/storage/storage.yaml b/kustomize/dependencies/storage/storage.yaml new file mode 100644 index 0000000000..d0ac2d0390 --- /dev/null +++ b/kustomize/dependencies/storage/storage.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: flyte +spec: + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + volumes: + - name: minio-storage + emptyDir: {} + containers: + - image: "minio/minio:RELEASE.2019-06-04T01-15-58Z" + name: minio + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: miniostorage + args: + - server + - /data + ports: + - containerPort: 9000 + name: minio + volumeMounts: + - name: minio-storage + mountPath: /data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: flyte +spec: + externalName: minio + ports: + - port: 9000 + selector: + app: minio +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: minio + namespace: flyte + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" +spec: + rules: + - http: + paths: + - path: /minio + backend: + serviceName: minio + servicePort: 9000 diff --git a/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml new file mode 100644 index 0000000000..31abf6ec3c --- /dev/null +++ b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteadmin + namespace: flyte +spec: + template: + spec: + initContainers: + - name: check-db-ready + image: postgres:10.1 + command: ['sh', '-c', + 'until pg_isready -h postgres -p 5432; + do echo waiting for database; sleep 2; done;'] + - name: run-migrations + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", + "migrate", "run"] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: seed-projects + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", + "migrate", "seed-projects", "flytesnacks", "flytetester"] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + containers: + - name: flyteadmin + resources: + limits: + memory: "200Mi" + cpu: "0.1" + ephemeral-storage: "100Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: flyteadmin + namespace: flyte +spec: + ports: + - name: redoc + protocol: TCP + port: 87 + targetPort: 8087 diff --git a/kustomize/overlays/sandbox/admindeployment/clusterresource-templates/aa_namespace.yaml b/kustomize/overlays/sandbox/admindeployment/clusterresource-templates/aa_namespace.yaml new file mode 100644 index 0000000000..3075aa9f9e --- /dev/null +++ b/kustomize/overlays/sandbox/admindeployment/clusterresource-templates/aa_namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ namespace }} +spec: + finalizers: + - kubernetes diff --git a/kustomize/overlays/sandbox/admindeployment/cron.yaml b/kustomize/overlays/sandbox/admindeployment/cron.yaml new file mode 100644 index 0000000000..f44221e1cb --- /dev/null +++ b/kustomize/overlays/sandbox/admindeployment/cron.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: syncresources + namespace: flyte +spec: + schedule: "*/1 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: sync-cluster-resources + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] + volumeMounts: + - name: resource-templates + mountPath: /etc/flyte/clusterresource/templates + - name: config-volume + mountPath: /etc/flyte/config + volumes: + - name: resource-templates + configMap: + name: clusterresource-template + - name: config-volume + configMap: + name: flyte-admin-config + restartPolicy: OnFailure diff --git a/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml b/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml new file mode 100644 index 0000000000..4baeda7fa0 --- /dev/null +++ b/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml @@ -0,0 +1,102 @@ +logger: + show-source: true + level: 5 +application: + httpPort: 8088 + grpcPort: 8089 +flyteadmin: + roleNameKey: "iam.amazonaws.com/role" + profilerPort: 10254 + metricsScope: "flyte:" + metadataStoragePrefix: + - "metadata" + - "admin" + testing: + host: http://flyteadmin +database: + port: 5432 + username: postgres + host: postgres + dbname: postgres + options: sslmode=disable +scheduler: + eventScheduler: + scheme: "local" + region: "us-east-1" + scheduleRole: "arn:aws:iam::abc123:role/my-schedule-role" + targetName: "arn:aws:sqs:my-region:abc123:my-queue" + workflowExecutor: + scheme: "local" + region: "us-east-1" + scheduleQueueName: "won't-work-locally" + accountId: "173840052742" +remoteData: + region: "us-east-1" + scheme: "local" + signedUrls: + durationMinutes: 3 +notifications: + type: local + region: "us-east-1" + publisher: + topicName: "foo" + processor: + queueName: "queue" + accountId: "bar" +queues: + executionQueues: + - parent: "FlyteNonProductionTaskCPU-staging-iad" + child: "FlyteNonProductionSubTaskCPU-staging-iad" + attributes: + - all +workflowConfigs: + - project: flytekit + tags: + - all + - project: "flytetester" + domain: "development" + workflowName: "app-workflows-rich-workflow-rich-workflow" + tags: + - all + - project: "flytetester" + domain: "development" + workflowName: "app-workflows-batch-workflow-batch-tasks-workflow" + tags: + - all + - tags: + - all +storage: + type: minio + connection: + access-key: minio + auth-type: accesskey + secret-key: miniostorage + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + container: "my-s3-bucket" +task_resources: + defaults: + cpu: 100m + memory: 100Mi + storage: 5Mi + limits: + cpu: 2 + memory: 2G + storage: 20Mi +task_type_whitelist: + spark: + - project: flytekit + - project: flytetester +domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + - id: domain + name: domain +cluster_resources: + templatePath: "/etc/flyte/clusterresource/templates" + refresh: 5m diff --git a/kustomize/overlays/sandbox/admindeployment/kustomization.yaml b/kustomize/overlays/sandbox/admindeployment/kustomization.yaml new file mode 100644 index 0000000000..55186e329c --- /dev/null +++ b/kustomize/overlays/sandbox/admindeployment/kustomization.yaml @@ -0,0 +1,21 @@ +bases: +- ../../../base/admindeployment + +namespace: flyte + +resources: +- cron.yaml + +configMapGenerator: +# the main admin configmap +- name: flyte-admin-config + files: + - flyteadmin_config.yaml +# cluster resource templates +- name: clusterresource-template + files: +# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". + - clusterresource-templates/aa_namespace.yaml + +patches: +- admindeployment.yaml diff --git a/kustomize/overlays/sandbox/console/console.yaml b/kustomize/overlays/sandbox/console/console.yaml new file mode 100644 index 0000000000..9261654671 --- /dev/null +++ b/kustomize/overlays/sandbox/console/console.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteconsole + namespace: flyte +spec: + template: + spec: + containers: + - name: flyteconsole + resources: + limits: + memory: "150Mi" + cpu: "0.1" + ephemeral-storage: "100Mi" diff --git a/kustomize/overlays/sandbox/console/kustomization.yaml b/kustomize/overlays/sandbox/console/kustomization.yaml new file mode 100644 index 0000000000..49f3deb5a9 --- /dev/null +++ b/kustomize/overlays/sandbox/console/kustomization.yaml @@ -0,0 +1,5 @@ +bases: +- ../../../base/console + +patches: +- console.yaml diff --git a/kustomize/overlays/sandbox/datacatalog/datacatalog.yaml b/kustomize/overlays/sandbox/datacatalog/datacatalog.yaml new file mode 100644 index 0000000000..40eb2cefec --- /dev/null +++ b/kustomize/overlays/sandbox/datacatalog/datacatalog.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datacatalog + namespace: flyte +spec: + template: + spec: + initContainers: + - name: check-db-ready + image: postgres:10.1 + command: ['sh', '-c', + 'until pg_isready -h postgres -p 5432; + do echo waiting for database; sleep 2; done;'] + volumeMounts: + - name: config-volume + mountPath: /etc/datacatalog/config + containers: + - name: datacatalog + resources: + limits: + memory: "200Mi" + cpu: "0.1" + ephemeral-storage: "100Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: datacatalog + namespace: flyte diff --git a/kustomize/overlays/sandbox/datacatalog/datacatalog_config.yaml b/kustomize/overlays/sandbox/datacatalog/datacatalog_config.yaml new file mode 100644 index 0000000000..7812ba566d --- /dev/null +++ b/kustomize/overlays/sandbox/datacatalog/datacatalog_config.yaml @@ -0,0 +1,28 @@ +logger: + show-source: true + level: 5 +datacatalog: + storage-prefix: metadata/datacatalog + metrics-scope: "datacatalog" + profiler-port: 10254 +application: + grpcPort: 8089 +storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + cache: + max_size_mbs: 10 + target_gc_percent: 100 + container: my-container + type: minio +database: + port: 5432 + username: postgres + host: postgres + dbname: datacatalog + options: sslmode=disable diff --git a/kustomize/overlays/sandbox/datacatalog/kustomization.yaml b/kustomize/overlays/sandbox/datacatalog/kustomization.yaml new file mode 100644 index 0000000000..9876c444a0 --- /dev/null +++ b/kustomize/overlays/sandbox/datacatalog/kustomization.yaml @@ -0,0 +1,12 @@ +bases: +- ../../../base/datacatalog + +namespace: flyte + +configMapGenerator: +- name: datacatalog-config + files: + - datacatalog_config.yaml + +patches: +- datacatalog.yaml diff --git a/kustomize/overlays/sandbox/flyte/kustomization.yaml b/kustomize/overlays/sandbox/flyte/kustomization.yaml new file mode 100644 index 0000000000..61f5016db3 --- /dev/null +++ b/kustomize/overlays/sandbox/flyte/kustomization.yaml @@ -0,0 +1,19 @@ +bases: +# global resources +- ../../../base/namespace +- ../../../dependencies/database +- ../../../dependencies/storage + +# user plane / control plane resources +- ../../../base/ingress +- ../../../dependencies/contour_ingress_controller +- ../admindeployment +- ../datacatalog +- ../console + +# data plane resources +- ../../../base/wf_crd +- ../../../base/operators/spark +- ../../../base/adminserviceaccount +- ../propeller +- ../redis diff --git a/kustomize/overlays/sandbox/propeller/config.yaml b/kustomize/overlays/sandbox/propeller/config.yaml new file mode 100644 index 0000000000..5a2d12e7fc --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/config.yaml @@ -0,0 +1,54 @@ +propeller: + metadata-prefix: metadata/propeller + workers: 4 + max-workflow-retries: 30 + workflow-reeval-duration: 30s + downstream-eval-duration: 30s + limit-namespace: "all" + prof-port: 10254 + metrics-prefix: flyte + enable-admin-launcher: true + leader-election: + lock-config-map: + name: propeller-leader + namespace: flyte + enabled: true + lease-duration: 15s + renew-deadline: 10s + retry-period: 2s + queue: + type: batch + batching-interval: 2s + batch-size: -1 + queue: + type: bucket + rate: 10 + capacity: 100 + sub-queue: + type: bucket + rate: 10 + capacity: 100 +logger: + show-source: true + level: 5 +storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + type: minio + container: my-s3-bucket +event: + type: admin + rate: 500 + capacity: 1000 +admin: + endpoint: flyteadmin:81 + insecure: true +catalog-cache: + endpoint: datacatalog:89 + type: datacatalog + insecure: true diff --git a/kustomize/overlays/sandbox/propeller/kustomization.yaml b/kustomize/overlays/sandbox/propeller/kustomization.yaml new file mode 100644 index 0000000000..f2de35f814 --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/kustomization.yaml @@ -0,0 +1,35 @@ +bases: +- ../../../base/propeller + +namespace: flyte + +configMapGenerator: +# the main propeller configmap +- name: flyte-propeller-config + files: + - config.yaml +# the plugin-configmap +- name: flyte-plugin-config + files: + - plugins/config.yaml +# a configmap for each plugin +- name: flyte-spark-config + files: + - plugins/spark/config.yaml +- name: flyte-waitable-config + files: + - plugins/waitable/config.yaml +- name: flyte-container-config + files: + - plugins/container/config.yaml +- name: flyte-qubole-config + files: + - plugins/qubole/config.yaml + +patches: +- propeller.yaml +# add the volumemount for each plugin configmap +- plugins/spark/propeller-patch.yaml +- plugins/waitable/propeller-patch.yaml +- plugins/container/propeller-patch.yaml +- plugins/qubole/propeller-patch.yaml diff --git a/kustomize/overlays/sandbox/propeller/plugins/config.yaml b/kustomize/overlays/sandbox/propeller/plugins/config.yaml new file mode 100644 index 0000000000..564d9d41c8 --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/plugins/config.yaml @@ -0,0 +1,15 @@ +plugins: + enabled-plugins: + - container + - spark + - waitable + - hiveExecutor + - sidecar + logs: + kubernetes-enabled: true + kubernetes-url: "http://localhost:30082" + k8s: + default-env-vars: + - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" + - FLYTE_AWS_ACCESS_KEY_ID: minio + - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage diff --git a/kustomize/overlays/sandbox/propeller/plugins/container/config.yaml b/kustomize/overlays/sandbox/propeller/plugins/container/config.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kustomize/overlays/sandbox/propeller/plugins/container/propeller-patch.yaml b/kustomize/overlays/sandbox/propeller/plugins/container/propeller-patch.yaml new file mode 100644 index 0000000000..fe9e305e28 --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/plugins/container/propeller-patch.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + volumes: + - name: container-config-volume + configMap: + name: flyte-container-config + containers: + - name: flytepropeller + volumeMounts: + - name: container-config-volume + mountPath: /etc/flyte/config-container diff --git a/kustomize/overlays/sandbox/propeller/plugins/qubole/config.yaml b/kustomize/overlays/sandbox/propeller/plugins/qubole/config.yaml new file mode 100644 index 0000000000..f7d776659c --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/plugins/qubole/config.yaml @@ -0,0 +1,10 @@ +plugins: + qubole: + # Either create this file with your username with the real token, or set the QUBOLE_API_KEY environment variable + # See the secrets_manager.go file in the plugins repo for usage. Since the dev/test deployment of + # this has a dummy QUBOLE_API_KEY env var built in, this fake path won't break anything. + quboleTokenPath: "/Path/To/QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 10 diff --git a/kustomize/overlays/sandbox/propeller/plugins/qubole/propeller-patch.yaml b/kustomize/overlays/sandbox/propeller/plugins/qubole/propeller-patch.yaml new file mode 100644 index 0000000000..cd85986073 --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/plugins/qubole/propeller-patch.yaml @@ -0,0 +1,20 @@ +# This file is only for volume mounts. The configmap itself that's being mounted is sufficiently different that +# there's no benefit to having it in this folder, since the entire thing gets overridden anyways. +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + volumes: + - name: qubole-config-volume + configMap: + name: flyte-qubole-config + containers: + - name: flytepropeller + volumeMounts: + - name: qubole-config-volume + mountPath: /etc/flyte/config-qubole diff --git a/kustomize/overlays/sandbox/propeller/plugins/spark/config.yaml b/kustomize/overlays/sandbox/propeller/plugins/spark/config.yaml new file mode 100644 index 0000000000..c1ffb9209a --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/plugins/spark/config.yaml @@ -0,0 +1,16 @@ +plugins: + spark: + spark-config-default: + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" + - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.blacklist.enabled: "true" + - spark.blacklist.timeout: "5m" + - spark.task.maxfailures: "8" diff --git a/kustomize/overlays/sandbox/propeller/plugins/spark/propeller-patch.yaml b/kustomize/overlays/sandbox/propeller/plugins/spark/propeller-patch.yaml new file mode 100644 index 0000000000..a1c2aacd10 --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/plugins/spark/propeller-patch.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + volumes: + - name: spark-config-volume + configMap: + name: flyte-spark-config + containers: + - name: flytepropeller + volumeMounts: + - name: spark-config-volume + mountPath: /etc/flyte/config-spark diff --git a/kustomize/overlays/sandbox/propeller/plugins/waitable/config.yaml b/kustomize/overlays/sandbox/propeller/plugins/waitable/config.yaml new file mode 100644 index 0000000000..e00c85a0cc --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/plugins/waitable/config.yaml @@ -0,0 +1,3 @@ +plugins: + waitable: + console-uri: http://localhost:30081/console diff --git a/kustomize/overlays/sandbox/propeller/plugins/waitable/propeller-patch.yaml b/kustomize/overlays/sandbox/propeller/plugins/waitable/propeller-patch.yaml new file mode 100644 index 0000000000..13cd9baeda --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/plugins/waitable/propeller-patch.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + volumes: + - name: waitable-config-volume + configMap: + name: flyte-waitable-config + containers: + - name: flytepropeller + volumeMounts: + - name: waitable-config-volume + mountPath: /etc/flyte/config-waitable diff --git a/kustomize/overlays/sandbox/propeller/propeller.yaml b/kustomize/overlays/sandbox/propeller/propeller.yaml new file mode 100644 index 0000000000..498055e6fc --- /dev/null +++ b/kustomize/overlays/sandbox/propeller/propeller.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + containers: + - name: flytepropeller + env: + - name: QUBOLE_API_KEY + value: notarealkey + resources: + limits: + memory: "100Mi" + cpu: "0.1" + ephemeral-storage: "100Mi" diff --git a/kustomize/overlays/sandbox/redis/kustomization.yaml b/kustomize/overlays/sandbox/redis/kustomization.yaml new file mode 100644 index 0000000000..12ec57311d --- /dev/null +++ b/kustomize/overlays/sandbox/redis/kustomization.yaml @@ -0,0 +1,5 @@ +bases: +- ../../../dependencies/redis + +patches: +- storage.yaml diff --git a/kustomize/overlays/sandbox/redis/storage.yaml b/kustomize/overlays/sandbox/redis/storage.yaml new file mode 100644 index 0000000000..0c5aed931b --- /dev/null +++ b/kustomize/overlays/sandbox/redis/storage.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: flyte +spec: + template: + spec: + volumes: + - name: redis-data + emptyDir: {} diff --git a/kustomize/overlays/test/flyte/kustomization.yaml b/kustomize/overlays/test/flyte/kustomization.yaml new file mode 100644 index 0000000000..0aab12e539 --- /dev/null +++ b/kustomize/overlays/test/flyte/kustomization.yaml @@ -0,0 +1,13 @@ +# tests use mostly the dev configs +# they do not need the console +bases: +- ../../../base/namespace +- ../../../base/wf_crd +- ../../../base/adminserviceaccount +- ../../../dependencies/database +- ../../../dependencies/storage +- ../propeller +- ../../sandbox/admindeployment +- ../../../dependencies/contour_ingress_controller +- ../../../base/ingress +- ../redis diff --git a/kustomize/overlays/test/propeller/config.yaml b/kustomize/overlays/test/propeller/config.yaml new file mode 100644 index 0000000000..99658f144e --- /dev/null +++ b/kustomize/overlays/test/propeller/config.yaml @@ -0,0 +1,50 @@ +propeller: + metadata-prefix: metadata/propeller + workers: 4 + max-workflow-retries: 30 + workflow-reeval-duration: 30s + downstream-eval-duration: 30s + limit-namespace: "all" + prof-port: 10254 + metrics-prefix: flyte + enable-admin-launcher: true + leader-election: + lock-config-map: + name: propeller-leader + namespace: flyte + enabled: true + lease-duration: 15s + renew-deadline: 10s + retry-period: 2s + queue: + type: batch + batching-interval: 2s + batch-size: -1 + queue: + type: bucket + rate: 10 + capacity: 100 + sub-queue: + type: bucket + rate: 10 + capacity: 100 +logger: + show-source: true + level: 5 +storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + type: minio + container: my-s3-bucket +event: + type: admin + rate: 500 + capacity: 1000 +admin: + endpoint: flyteadmin:81 + insecure: true diff --git a/kustomize/overlays/test/propeller/kustomization.yaml b/kustomize/overlays/test/propeller/kustomization.yaml new file mode 100644 index 0000000000..b6a993b57c --- /dev/null +++ b/kustomize/overlays/test/propeller/kustomization.yaml @@ -0,0 +1,17 @@ +bases: +- ../../../base/propeller + +namespace: flyte + +configMapGenerator: +# the main propeller configmap +- name: flyte-propeller-config + files: + - config.yaml +# the plugin-configmap +- name: flyte-plugin-config + files: + - plugins/config.yaml + +patches: +- propeller.yaml diff --git a/kustomize/overlays/test/propeller/plugins/config.yaml b/kustomize/overlays/test/propeller/plugins/config.yaml new file mode 100644 index 0000000000..9d09f4a63f --- /dev/null +++ b/kustomize/overlays/test/propeller/plugins/config.yaml @@ -0,0 +1,12 @@ +plugins: + enabled-plugins: + - container + - sidecar + logs: + kubernetes-enabled: true + kubernetes-url: "http://localhost:30082" + k8s: + default-env-vars: + - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" + - FLYTE_AWS_ACCESS_KEY_ID: minio + - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage diff --git a/kustomize/overlays/test/propeller/propeller.yaml b/kustomize/overlays/test/propeller/propeller.yaml new file mode 100644 index 0000000000..cde598b46d --- /dev/null +++ b/kustomize/overlays/test/propeller/propeller.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + containers: + - name: flytepropeller + resources: + limits: + memory: "100Mi" + cpu: "0.1" + ephemeral-storage: "100Mi" diff --git a/kustomize/overlays/test/redis/kustomization.yaml b/kustomize/overlays/test/redis/kustomization.yaml new file mode 100644 index 0000000000..12ec57311d --- /dev/null +++ b/kustomize/overlays/test/redis/kustomization.yaml @@ -0,0 +1,5 @@ +bases: +- ../../../dependencies/redis + +patches: +- storage.yaml diff --git a/kustomize/overlays/test/redis/storage.yaml b/kustomize/overlays/test/redis/storage.yaml new file mode 100644 index 0000000000..0c5aed931b --- /dev/null +++ b/kustomize/overlays/test/redis/storage.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: flyte +spec: + template: + spec: + volumes: + - name: redis-data + emptyDir: {} diff --git a/rsts/administrator/architecture.rst b/rsts/administrator/architecture.rst new file mode 100644 index 0000000000..6c24c13940 --- /dev/null +++ b/rsts/administrator/architecture.rst @@ -0,0 +1,93 @@ +.. _architecture-overview: + +############ +Architecture +############ + +This document aims to demystify how Flyte's major components ``FlyteIDL``, ``FlyteKit``, ``FlyteCLI``, ``FlyteConsole``, ``FlyteAdmin``, ``FlytePropeller``, and ``FlytePlugins`` fit together at a high level. + +FlyteIDL +======== + +In Flyte, entities like "Workflows", "Tasks", "Launch Plans", and "Schedules" are recognized by multiple system components. In order for components to communicate effectively, they need a shared understanding about the structure of these entities. + +The Flyte IDL (Interface Definition Language) is where shared Flyte entities are defined. This IDL also defines the RPC service definition for the `core Flyte API `_. + +FlyteIDL uses the `protobuf `_ schema to describe entities. Clients are generated for Python, Golang, and JavaScript and imported by Flyte components. + + +Planes ✈️ +========= + +Flyte components are separated into 3 logical planes. The planes are summarized here and explained in further detail below. The goal is that any of these planes can be replaced by an alternate implementation. + ++-------------------+---------------------------------------------------------------------------------------------------------------+ +| **User Plane** | The User Plane consists of all user tools that assist in interacting with the core Flyte API. | +| | These tools include the FlyteConsole, FlyteKit, and FlyteCLI. | ++-------------------+---------------------------------------------------------------------------------------------------------------+ +| **Control Plane** | The Control Plane implements the core Flyte API. | +| | It serves all client requests coming from the User Plane. | +| | It stores information such as current and past running workflows, and provides that information upon request. | +| | It also accepts requests to execute workflows, but offloads the work to the Data Plane. | ++-------------------+---------------------------------------------------------------------------------------------------------------+ +| **Data Plane** | The sole responsibility of the the Data Plane is to fulfill workflows. | +| | It accepts workflow requests from the Control Plane and guides the workflow to completion, | +| | launching tasks on a cluster of machines as necessary based on the workflow graph. | +| | It sends status events back to the control plane so the information can be stored and surfaced to end-users. | ++-------------------+---------------------------------------------------------------------------------------------------------------+ + +User Plane +---------- + +In Flyte, workflows are represented as a Directed Acyclic Graph (DAG) of tasks. While this representation is logical for services, managing workflow DAGs in this format is a tedious exercise for humans. The Flyte User Plane provides tools to create, manage, and visualize workflows in a format that is easily digestible to users. + +These tools include: + +FlyteKit + FlyteKit is an SDK that helps users design new workflows using the Python programming language. FlyteKit can parse the python code, compile it into a valid Workflow DAG, and submit it to Flyte to be executed. + +FlyteConsole + Flyte console provides the Web interface for Flyte. Users and administrators can use the console to view workflows, launch plans, schedules, tasks, and individual task executions. The console provides tools to visualize workflows, and surfaces relevant logs for debugging failed tasks. + +FlyteCLI + Flyte Command Line Interface provides interactive access to Flyte to launch and access Flyte workflows via terminal. + + +Control Plane +------------- + +The Control Plane supports the core REST/gRPC API defined in FlyteIDL. User Plane tools like FlyteConsole and FlyteKit contact the control plane on behalf of users to store and retrieve information. + +Currently, the entire control plane is handled by a single service called **FlyteAdmin**. + +FlyteAdmin is stateless. It processes requests to create entities like Tasks, Workflows, and Schedules by persisting data in a relational database. + +While FlyteAdmin serves the Workflow Exeuction API, it does not, itself, execute workflows. To launch workflow executions, FlyteAdmin sends the workflow DAG off to the DataPlane. For added scalability and fault-tolerance, FlyteAdmin can be configured to load-balance workflows across multiple isolated data-plane clusters. + + +Data Plane +---------- + +The Data Plane is the engine that accepts DAGs, and fulfills workflow executions by launching tasks in the order defined by the graph. Requests to the Data Plane generally come via the control plane, and not from end-users. + +In order to support compute-intensive workflows at massive scale, the Data Plane needs to launch containers on a cluster of machines. The current implementation leverages `kubernetes `_ for cluster management. + +Unlike the user-facing control-plane, the Data Plane does not expose a traditional REST/gRPC API. To launch an execution in the Data Plane, you create a “flyteworkflow” resource in kubernetes. +A “flyteworkflow” is a kubernetes `Custom Resource `_ (CRD) created by our team. This custom resource represents the flyte workflow DAG. + +The core state machine that processes flyteworkflows is worker we call **FlytePropeller**. + +FlytePropeller leverages the kubernetes `operator pattern `_. It polls the kubernetes API, looking for newly created flyteworkflow resources. FlytePropeller understands the workflow DAG, and launches the appropriate kubernetes pods as needed to complete tasks. It periodically checks for completed tasks, launching downstream tasks until the workflow is complete. + +**Plugins** + +Each task in a flyteworkflow DAG has a specified **type**. The logic for fulfilling a task is determined by its task type. +In the most basic case, FlytePropeller launches a single kubernetes pod to fulfill a task. +More complex task types require workloads to be distributed across hundreds of pods. + +The type-specific task logic is separated into isolated code modules that we call **plugins**. +Each task type has an associated plugin that is responsible for handling tasks of its type. +For each task in a workflow, FlytePropeller activates the appropriate plugin based on the task type in order to fullfill the task. + +The Flyte team has pre-built plugins for Hive, Spark, and AWS Batch, and more. +To support new use-cases, developers can create their own plugins and bundle them in their FlytePropeller deployment. diff --git a/rsts/administrator/index.rst b/rsts/administrator/index.rst new file mode 100644 index 0000000000..f99c047596 --- /dev/null +++ b/rsts/administrator/index.rst @@ -0,0 +1,17 @@ +.. _administrator: + +Administrator Docs +================== + +This section provides details on how to manage and scale +Flyte deployments within your organization. To manage Flyte, we will first +provide an overview of the architecture, follow that up with an installation +section and then describe how to manage and scale it for your users or yourself. + +.. toctree:: + :maxdepth: 2 + :caption: Administrator Documentation + :name: admindoc + + architecture + install/index diff --git a/rsts/administrator/install/configure/admin.rst b/rsts/administrator/install/configure/admin.rst new file mode 100644 index 0000000000..d5be502e9c --- /dev/null +++ b/rsts/administrator/install/configure/admin.rst @@ -0,0 +1,12 @@ +.. _config-admin: + +############################# +FlyteAdmin Configuration +############################# + +.. _admin-config-example: + +Example config +============== + +.. literalinclude:: ../../../../kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml diff --git a/rsts/administrator/install/configure/common.rst b/rsts/administrator/install/configure/common.rst new file mode 100644 index 0000000000..887b6e8594 --- /dev/null +++ b/rsts/administrator/install/configure/common.rst @@ -0,0 +1,59 @@ +.. _common-config: + +################################################### +Common configuration across all backend components +################################################### +All backend Flyte components are written in GoLang and use `flytestdlib `_. Flytestdlib uses the common configuration system and is shared across the entire +flyte-backend. The configuration used to configure the logging and metadata storage systems. + +Logger Configuration +==================== +Logger can be configured using a common configuration that is shared across all components. The +variables available for configuration are described in the **Config** struct in the `source code `_. Documentation for this is +available @ `godoc:flytestdlib/logger `_. + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|logger.formatter.type |string |Sets logging format type. (default "json") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|logger.level |int |Sets the minimum logging level. (default 4) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|logger.mute |Bool |Mutes all logs regardless of severity. Intended for benchmarks/tests only. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|logger.show-source |Bool |Includes source code location in logs. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +Metadata storage configuration +=============================== +Metadata Storage can be configured using a common configuration that is shared across all components. The +variables available for configuration are described in the **Config** struct in the `code `_. Documentation for this is +available @ `godoc:flytestdlib/storage `_ + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.cache.max_size_mbs |int |Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.cache.target_gc_percent |int |Sets the garbage collection target percentage. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.connection.access-key |string |Access key to use. Only required when authtype is set to accesskey. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.connection.auth-type |string |Auth Type to use [iam, accesskey]. (default "iam") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.connection.disable-ssl |Disables |SSL connection. Should only be used for development. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.connection.endpoint |string |URL for storage client to connect to. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.connection.region |string |Region to connect to. (default "us-east-1") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.connection.secret-key |string |Secret to use when accesskey is set. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.container |string |Initial container to create -if it doesn't exist-.' | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.limits.maxDownloadMBs |int |Maximum allowed download size (in MBs) per call. (default 2) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|storage.type |string |Sets the type of storage to configure [s3/minio/local/mem]. (default "s3") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + diff --git a/rsts/administrator/install/configure/index.rst b/rsts/administrator/install/configure/index.rst new file mode 100644 index 0000000000..8a878226c3 --- /dev/null +++ b/rsts/administrator/install/configure/index.rst @@ -0,0 +1,15 @@ +.. _configure: + +################################ +Configure Flyte backend +################################ +Flyte has been designed to be extremely configurable. The entire backend has been written in Golang and uses `flytestdlib `_ module to manage configuration. + +.. toctree:: + :caption: Configuring Flyte Backend + :name: configuretoc + + common + propeller + plugins + admin diff --git a/rsts/administrator/install/configure/plugins.rst b/rsts/administrator/install/configure/plugins.rst new file mode 100644 index 0000000000..e0bc8b999d --- /dev/null +++ b/rsts/administrator/install/configure/plugins.rst @@ -0,0 +1,73 @@ +.. _plugins-config: + +######################### +Plugin Configuration +######################### + +.. _plugins-config-enable: + +Controlling the plugins available +================================= + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|plugins.enabled-plugins |strings |List of enabled plugins, default value is to enable all plugins. (default [*]) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +.. _plugins-config-log-links: + +Controlling log-links for user containers +========================================= + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|plugins.logs.cloudwatch-enabled |Bool |Cloudwatch Logging | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|plugins.logs.cloudwatch-log-group |string |Log group to which streams are associated. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|plugins.logs.cloudwatch-region |string |AWS region in which Cloudwatch logs are stored. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|plugins.logs.kubernetes-enabled |Bool |Kubernetes Logging | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|plugins.logs.kubernetes-url |string |Console URL for Kubernetes logs | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +.. _plugins-config-k8s-defaults: + +Default values to provide kubernetes container executions +========================================================= + + +.. _plugins-config-per-plugin: + +Individual plugins configuration +================================ +Refer to plugin documentation for understanding the configuration alternatives + + +.. _plugins-config-example: + +Example Configuration +===================== + +Top level Plugin configuration example +-------------------------------------- + +.. literalinclude:: ../../../../kustomize/overlays/sandbox/propeller/plugins/config.yaml + +Individual plugin configuration example +--------------------------------------- + +Spark + +.. literalinclude:: ../../../../kustomize/overlays/sandbox/propeller/plugins/spark/config.yaml + +Qubole + +.. literalinclude:: ../../../../kustomize/overlays/sandbox/propeller/plugins/qubole/config.yaml + +Waitable + +.. literalinclude:: ../../../../kustomize/overlays/sandbox/propeller/plugins/waitable/config.yaml diff --git a/rsts/administrator/install/configure/propeller.rst b/rsts/administrator/install/configure/propeller.rst new file mode 100644 index 0000000000..6407ca2c48 --- /dev/null +++ b/rsts/administrator/install/configure/propeller.rst @@ -0,0 +1,195 @@ +.. _propeller-config: + +######################### +Propeller Configuration +######################### + +Admin Service Configuration +=========================== +Propeller uses the following configuration to talk with FlyteAdmin service (control plane). It needs to talk to FlyteAdmin Service for, + - Publishing events of progress in workflow execution + - Launching new launch plan executions as child executions + + Propeller assumes a grpc endpoint available on the flyteadmin service, which is defined in flyteidl + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|admin.endpoint |string |For admin types, specify where the uri of the service is located. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|admin.insecure |Bool |Use insecure connection. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +Event configuration +=================== +Propeller publishes events for every workflow execution and to prevent a large request spikes to FlyteAdmin service has a configurable tocken-bucket implementation. It can be configured using these +parameters. + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|event.capacity |int |The max bucket size for event recording tokens. (default 1000) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|event.file-path |string |For file types, specify where the file should be located. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|event.rate |int |Max rate at which events can be recorded per second. (default 500) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|event.type |string |Sets the type of EventSink to configure [log/admin/file]. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Propeller External Workflow Launcher configuration +================================================== +Propeller can launch child-workflow/launch plan executions using a configurable flyteadmin service endpoint. + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.admin-launcher.burst |int |Maximum burst for throttle (default 10) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.admin-launcher.cacheSize |int |Maximum cache in terms of number of items stored. (default 10000) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.admin-launcher.tps |int |The maximum number of transactions per second to flyte admin from this client. (default 5) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.enable-admin-launcher |Bool |remote Workflow launcher to Admin | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Catalog Service configuration +======================================= +The Catalog service is an optional add-on to Flyte. It enables various features - refer to :ref:`Catalog Service ` to understand the usecases. If available, it +can be configured using. + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.catalog-cache.endpoint |string |Endpoint for catalog service | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.catalog-cache.type |string |Catalog Implementation to use (default "noop") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Propeller Core Configuration +============================ +This is the core configuration for propeller. It controls, + - Garbage collection + - Scale in terms of number of workers + - namespaces to observe for new workflows + - handling of system failures + - and other system level configuration + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.downstream-eval-duration |string |Frequency of re-evaluating downstream tasks (default "60s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.gc-interval |string |Run periodic GC every 30 minutes (default "30m") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.kube-config |string |Path to kubernetes client config file. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.limit-namespace |string |Namespaces to watch for this propeller (default "all") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.master |string | . | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.max-ttl-hours |int |Maximum number of hours a completed workflow should be retained. Number between 1-23 hours (default 23) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.max-workflow-retries |int |Maximum number of retries per workflow (default 50) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.metadata-prefix |string |MetadataPrefix should be used if all the metadata for Flyte executions should be stored under a specific prefix in CloudStorage. If not specified, the data will be stored in the base container directly.| ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.metrics-prefix |string |An optional prefix for all published metrics. (default "flyte:") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.prof-port |string |Profiler port (default "10254") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.publish-k8s-events |Bool |events publishing to K8s events API. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.workers |int |Number of threads to process workflows (default 2) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.workflow-reeval-duration |string |Frequency of re-evaluating workflows (default "30s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +Propeller Leader election configuration +======================================= + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.leader-election.enabled |Bool |s/Disables leader election. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.leader-election.lease-duration |string |Duration that non-leader candidates will wait to force acquire leadership. This is measured against time of last observed ack. (default "15s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.leader-election.lock-config-map.Name |string |Name of config-map used for leader. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.leader-election.lock-config-map.Namespace|string |Namespace of config-map used for leader. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.leader-election.renew-deadline |string |Duration that the acting master will retry refreshing leadership before giving up. (default "10s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.leader-election.retry-period |string |Duration the LeaderElector clients should wait between tries of actions. (default "2s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +Propeller Queue configuration +============================= +Propeller uses workqueues to manage the event loops and these queues can be configured using the following parameters. + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.batch-size |int |Number of downstream triggered top-level objects to re-enqueue every duration. -1 indicates all available. (default -1) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.batching-interval |string |Duration for which downstream updates are buffered (default "1s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.queue.base-delay |string |base backoff delay for failure (default "10s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.queue.capacity |int |Bucket capacity as number of items (default 100) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.queue.max-delay |string |Max backoff delay for failure (default "10s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.queue.rate |int |Bucket Refill rate per second (default 10) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.queue.type |string |Type of RateLimiter to use for the WorkQueue (default "default") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.sub-queue.base-delay |string |base backoff delay for failure (default "10s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.sub-queue.capacity |int |Bucket capacity as number of items (default 100) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.sub-queue.max-delay |string |Max backoff delay for failure (default "10s") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.sub-queue.rate |int |Bucket Refill rate per second (default 10) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.sub-queue.type |string |Type of RateLimiter to use for the WorkQueue (default "default") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|propeller.queue.type |string |Type of composite queue to use for the WorkQueue (default "simple") | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Propeller Commandline only parameters +===================================== +These parameters are available when starting a propeller process. + ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|Config Val |Type |Description | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|alsologtostderr |Bool |log to standard error as well as files | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|config |string |config file (default is $HOME/config.yaml) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|errors.show-source |Bool |Indicates whether errors should include source code location information. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|kubeconfig |string |Paths to a kubeconfig. Only required if out-of-cluster. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|log_backtrace_at |Bool |trace location when logging hits line file:N, emit a stack trace (default :0) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|log_dir |string |If non-empty, write log files in this directory | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|logtostderr |Bool |log to standard error instead of files | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|master |string |The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster. | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|stderrthreshold |severity |logs at or above this threshold go to stderr (default 2) | ++---------------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +.. _propeller-config-example: + +Example Configuration +====================== + +.. literalinclude:: ../../../../kustomize/overlays/sandbox/propeller/config.yaml diff --git a/rsts/administrator/install/getting_started.rst b/rsts/administrator/install/getting_started.rst new file mode 100644 index 0000000000..17edee17b1 --- /dev/null +++ b/rsts/administrator/install/getting_started.rst @@ -0,0 +1,46 @@ +.. _getting_started: + +Getting Started +--------------- + +Prerequisites +************* + +Kubernetes and its ``kubectl`` client are the only strict prerequisites to installing Flyte. + +Kubernetes can be installed on your local machine to run Flyte locally, or in the cloud for a scalable multi-user setup. Some installation options are listed below. + +Local: + +- `Minikube `_ +- `Docker for Mac `_ + +Cloud Providers: + +- `AWS EKS `_ (Amazon) +- `GCP GKE `_ (Google) +- `Azure AKS `_ (Microsoft) + +Once you have kubernetes set up and can access it with ``kubectl cluster-info``, you're ready to deploy flyte. + +Flyte has a few different deployment configurations. We'll start with the easiest, and expand on it to increase scale and reliability. + + +Sandbox Deployment +****************** + +The simplest Flyte deployment is the "sandbox" deployment, which includes everything you need in order to use Flyte. The Flyte sandbox can be deployed with a single command :: + + kubectl create -f https://raw.githubusercontent.com/lyft/flyte/master/deployment/sandbox/flyte_generated.yaml + +This deployment uses a kubernetes `NodePort `_ for Flyte ingress. +Once deployed, you can access the Flyte console on any kubernetes node at ``http://{{ any kubernetes node }}:30081/console`` (note that it will take a moment to deploy). + +For local deployments, this endpoint is typically http://localhost:30081/console. + +WARNING: + The sandbox deployment is not well suited for production use. + Most importantly, Flyte needs access to an object store, and a PostgreSQL database. + In the sandbox deployment, the object store and PostgreSQL database are each installed as a single kubernetes pod. + These pods are sufficient for testing and playground purposes, but they not designed to handle production load. + Read on to learn how to configure Flyte for production. diff --git a/rsts/administrator/install/index.rst b/rsts/administrator/install/index.rst new file mode 100644 index 0000000000..d6f8579492 --- /dev/null +++ b/rsts/administrator/install/index.rst @@ -0,0 +1,16 @@ +.. _install: + +================ +Installing Flyte +================ + +This doc explains how to install Flyte, starting with the simplest method, and expanding on it to increase scale and reliability. + +.. toctree:: + :name: installtoc + + getting_started + production + multi_cluster + optional_components + configure/index diff --git a/rsts/administrator/install/multi_cluster.rst b/rsts/administrator/install/multi_cluster.rst new file mode 100644 index 0000000000..edaf120c06 --- /dev/null +++ b/rsts/administrator/install/multi_cluster.rst @@ -0,0 +1,123 @@ +.. _multi_cluster: + +Scaling Beyond Kubernetes +------------------------- + +As described in the high-level architecture doc, the Flyte Control Plane sends workflows off to the Data Plane for execution. +The Data Plane fulfills these workflows by launching pods in kubernetes. + +At some point, your total compute needs could exceed the limits of a single kubernetes cluster. +To address this, you can deploy the Data Plane to several isolated kubernetes clusters. +The Control Plane (FlyteAdmin) can be configured to load-balance workflows across these isolated Data Planes. +This protects you from a failure in a single kubernetes cluster, and increases scalability. + +First, you'll need to create additional kubernetes clusters. For this example, we'll assume you have 3 kubernetes clusters, and can access them all with ``kubectl``. We'll call these clusters "cluster1", "cluster2", and "cluster3". + +We want to deploy **just** the Data Plane to these clusters. To do this, we'll remove the DataPlane components from the ``flyte`` overlay, and create a new overlay containing **ony** the dataplane resources. + +Data Plane Deployment +********************* + +To create the "Data Plane only" overlay, lets make a ``dataplane`` subdirectory inside our main deployment directory (my-flyte-deployment). This directory will contain contain only the dataplane resources. :: + + mkdir dataplane + +Now, lets copy the ``flyte`` config into the dataplane config :: + + cp flyte/kustomization.yaml dataplane/kustomization.yaml + +Since the dataplane resources will live in the new deployment, they are no longer needed in the main ``flyte`` deployment. Remove the Data Plane resources from the flyte deploy by opening ``flyte/kustomization.yaml`` and removing everything in the ``DATA PLANE RESOURCES`` section. + +Likewise, the User Plane / Control Plane resources are not needed in the dataplane deployment. Remove these resources from the dataplane deploy by opening ``dataplane/kustomization.yaml`` and removing everything in the ``USER PLANE / CONTROL PLANE RESOURCES`` section. + +Now Run :: + + kustomize build dataplane > dataplane_generated.yaml + +You will notice that the only the Data Plane resources are included in this file. + +You can point your ``kubectl`` context at each of the 3 clusters and deploy the dataplane with :: + + kubectl apply -f dataplane_generated.yaml + +User and Control Plane Deployment +********************************* + +In order for FlyteAdmin to create "flyteworkflows" on the 3 remote clusters, it will need a secret ``token`` and ``cacert`` to access each cluster. + +Once you have deployed the dataplane as described above, you can retrieve the "token" and "cacert" by pointing your ``kubectl`` context each dataplane cluster and executing the following commands. + +:token: + ``kubectl get secrets -n flyte | grep flyteadmin-token | awk '{print $1}' | xargs kubectl get secret -n flyte -o jsonpath='{.data.token}'`` + +:cacert: + ``kubectl get secrets -n flyte | grep flyteadmin-token | awk '{print $1}' | xargs kubectl get secret -n flyte -o jsonpath='{.data.ca\.crt}'`` + +These credentials will need to be included in the Control Plane. Create a new file ``admindeployment/secrets.yaml`` that looks like this :: + + apiVersion: v1 + kind: Secret + metadata: + name: cluster_credentials + namespace: flyte + type: Opaque + data: + cluster_1_token: {{ cluster 1 token here }} + cluster_1_cacert: {{ cluster 1 cacert here }} + cluster_2_token: {{ cluster 2 token here }} + cluster_2_cacert: {{ cluster 2 cacert here }} + cluster_3_token: {{ cluster 3 token here }} + cluster_3_cacert: {{ cluster 3 cacert here }} + +Include the new ``secrets.yaml`` file in the ``admindeployment`` by opening ``admindeployment/kustomization.yaml`` and add the following line under ``resources:`` to include the secrets in the deploy :: + + - secrets.yaml + +Next, we'll need to attach these secrets to the FlyteAdmin pods so that FlyteAdmin can access them. Open ``admindeployment/deployment.yaml`` and add an entry under ``volumes:`` :: + + volumes: + - name: cluster_credentials + secret: + secretName: cluster_credentials + +Now look for the container labeled ``flyteadmin``. Add a ``volumeMounts`` to that section. :: + + volumeMounts: + - name: cluster_credentials + mountPath: /var/run/credentials + +This mounts the credentials inside the FlyteAdmin pods, however, FlyteAdmin needs to be configured to use these credentials. Open the ``admindeployment/configmap.yaml`` file and add a ``clusters`` key to the configmap, with an entry for each cluster :: + + clusters: + - name: "cluster_1" + endpoint: {{ your-cluster-1-kubeapi-endpoint.com }} + enabled: true + auth: + type: "file_path" + tokenPath: "/var/run/credentials/cluster_1_token" + certPath: "/var/run/credentials/cluster_1_cacert" + - name: "cluster_2" + endpoint: {{ your-cluster-2-kubeapi-endpoint.com }} + auth: + enabled: true + type: "file_path" + tokenPath: "/var/run/credentials/cluster_2_token" + certPath: "/var/run/credentials/cluster_2_cacert" + - name: "cluster_3" + endpoint: {{ your-cluster-3-kubeapi-endpoint.com }} + enabled: true + auth: + type: "file_path" + tokenPath: "/var/run/credentials/cluster_3_token" + certPath: "/var/run/credentials/cluster_3_cacert" + +Now re-run :: + + kustomize build flyte > flyte_generated.yaml + +You will notice that the Data Plane resources have been removed from the ``flyte_generated.yaml`` file, and your new configurations have been added. + +Deploy the user/control plane to one cluster (you could use one of 3 existing clusters, or an entirely separate cluster). :: + + kubectl apply -f flyte_generated.yaml + diff --git a/rsts/administrator/install/optional_components.rst b/rsts/administrator/install/optional_components.rst new file mode 100644 index 0000000000..2738102dd2 --- /dev/null +++ b/rsts/administrator/install/optional_components.rst @@ -0,0 +1,17 @@ +.. _install-dependencies: + +Optional Components +------------------- + +Scheduling +********** + +Scheduling component of Flyte allows workflows to run at a configured cadence. +Currently, Flyte does not have a built in cron style scheduler. +It uses external services to manage these schedules and launch executions of scheduled workflows at specific intervals. + +Data Catalog +------------ + +The Data Catalog component enables features like *Data Provenance*, *Data Lineage* and *Data Caching*. +Data Catalog is enabled in Flyte by default. diff --git a/rsts/administrator/install/production.rst b/rsts/administrator/install/production.rst new file mode 100644 index 0000000000..489df7defd --- /dev/null +++ b/rsts/administrator/install/production.rst @@ -0,0 +1,156 @@ +.. _production: + +Handling Production Load +------------------------ + +In order to handle production load, you'll want to replace the sandbox's object store and PostgreSQL database with production grade storage systems. To do this, you'll need to modify your Flyte configuration to remove the sandbox datastores and reference new ones. + +Flyte Configuration +******************* + +A Flyte deployment contains around 50 kubernetes resources. +The Flyte team has chosen to use the "kustomize" tool to manage these configs. +Take a moment to read the `kustomize docs `_. Understanding kustomize will be important to modifying Flyte configurations. + +The ``/kustomize`` directory in the `flyte repository `_ is designed for use with ``kustomize`` to tailor Flyte deployments to your needs. +Important subdirectories are described below. + +base + The `base directory `_ contains minimal configurations for each Flyte component. + +dependencies + The `dependencies directory `_ contains deploy configurations for components like ``PostgreSQL`` that Flyte depends on. + +These directories were designed so that you can modify them using ``kustomize`` to generate a custom Flyte deployment. +In fact, this is how we create the ``sandbox`` deployment. + +Understanding the sandbox deployment will help you to create your own custom deployments. + +Understanding the Sandbox +************************* + +The sandbox deployment is managed by a set of kustomize `overlays `_ that alter the ``base`` configurations to compose the sandbox deployment. + +The sandbox overlays live in the `kustomize/overlays/sandbox `_ directory. There are overlays for each component, and a "flyte" overlay that aggregates the components into a single deploy file. + +**Component Overlays** + For each modified component, there is an kustomize overlay at ``kustomize/overlays/sandbox/{{ component }}``. + The overlay will typically reference the ``base`` for that component, and modify it to the needs of the sandbox. + + Using kustomize "patches", we add or override specific configs from the ``base`` resources. For example, in the "console" overlay, we specify a patch in the `kustomization.yaml `_. This patch adds memory and cpu limits to the console deployment config. + + Each Flyte component requires at least one configuration file. The configuration files for each component live in the component overlay. For example, the FlyteAdmin config lives at `kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml `_. These files get included as Kubernetes configmaps and mounted into pods. + +**Flyte Overlay** + The ``flyte`` overlay is meant to aggregate the components into a single deployment file. + The `kustomization.yaml `_ in that directory lists the components to be included in the deploy. + + We run ``kustomize build`` against the ``flyte`` directory to generate the complete `sandbox deployment yaml `_ we used earlier to deploy Flyte sandbox. + +Creating Your Own Deployment +**************************** + +Before you create a custom deployment, you'll need to `install kustomize `_. + +The simplest way to create your own custom deployment is to clone the sandbox deploy and modify it to your liking. + +To do this, check out the ``flyte`` repo :: + + git clone https://github.com/lyft/flyte.git + +Copy the sandbox configuration to a new directory on your machine, and enter the new directory :: + + cp -r flyte/kustomize/overlays/sandbox my-flyte-deployment + cd my-flyte-deployment + +Since the ``base`` files are not in your local copy, you'll need to make some slight modifications to reference the ``base`` files from our GitHub repository. :: + + find . -name kustomization.yaml -print0 | xargs -0 sed -i.bak 's~../../../base~github.com/lyft/flyte/kustomize/base~' + find . -name kustomization.yaml -print0 | xargs -0 sed -i.bak 's~../../../dependencies~github.com/lyft/flyte/kustomize/dependencies~' + find . -name *.bak | xargs rm + +You should now be able to run kustomize against the ``flyte`` directory :: + + kustomize build flyte > flyte_generated.yaml + +This will generate a deployment file identical to the sandbox deploy, and place it in a file called ``flyte_generated.yaml`` + +Going Beyond the Sandbox +************************ + +Let's modify the sandbox deployment to use cloud providers for the database and object store. + +Production Grade Database +************************* + +The ``FlyteAdmin`` and ``DataCatalog`` components rely on PostgreSQL to store persistent records. + +In this section, we'll modify the Flyte deploy to use a remote PostgreSQL database instead. + +First, you'll need to set up a reliable PostgreSQL database. The easiest way achieve this is to use a cloud provider like AWS `RDS `_, GCP `Cloud SQL `_, or Azure `PostgreSQL `_ to manage the PostgreSQL database for you. Create one and make note of the username, password, endpoint, and port. + +Next, remove old sandbox database by opening up the ``flyte/kustomization.yaml`` file and deleting database component. :: + + - github.com/lyft/flyte/kustomize/dependencies/database + +With this line removed, you can re-run ``kustomize build flyte > flyte_generated.yaml`` and see that the the postgres deployment has been removed from the ``flyte_generated.yaml`` file. + +Now, let's re-configure ``FlyteAdmin`` to use the new database. +Edit the ``admindeployment/flyteadmin_config.yaml`` file, and change the ``storage`` key like so :: + + database: + host: {{ your-database.endpoint }} + port: {{ your database port }} + username: {{ your_database_username }} + password: {{ your_database_password }} + dbname: flyteadmin + +Do the same thing in ``datacatalog/datacatalog_config.yaml``, but use the dbname ``datacatalog`` :: + + database: + host: {{ your-database.endpoint }} + port: {{ your database port }} + username: {{ your_database_username }} + password: {{ your_database_password }} + dbname: datacatalog + +Note: *You can mount the database password into the pod and use the "passwordPath" config to point to a file on disk instead of specifying the password here* + +Next, remove the "check-db-ready" init container from `admindeployment/admindeployment.yaml `_. This check is no longer needed. + +Production Grade Object Store +***************************** + +``FlyteAdmin``, ``FlytePropeller``, and ``DataCatalog`` components rely on an Object Store to hold files. + +In this section, we'll modify the Flyte deploy to use `AWS S3 `_ for object storage. +The process for other cloud providers like `GCP GCS `_ should be similar. + +To start, `create an s3 bucket `_. + +Next, remove the old sandbox object store by opening up the ``flyte/kustomization.yaml`` file and deleting the storage line. :: + + - github.com/lyft/flyte/kustomize/dependencies/storage + +With this line gone, you can re-run ``kustomize build flyte > flyte_generated.yaml`` and see that the sandbox object store has been removed from the ``flyte_generated.yaml`` file. + +Next, open the configs ``admindeployment/flyteadmin_config.yaml``, ``propeller/config.yaml``, ``datacatalog/datacatalog_config.yaml`` and look for the ``storage`` configuration. + +Change the ``storage`` configuration in each of these configs to use your new s3 bucket like so :: + + storage: + type: s3 + container: {{ YOUR-S3-BUCKET }} + connection: + auth-type: accesskey + access-key: {{ YOUR_AWS_ACCESS_KEY }} + secret-key: {{ YOUR_AWS_SECRET_KEY }} + region: {{ YOUR-AWS-REGION }} + +Note: *To use IAM roles for authentication, switch to the "iam" auth-type.* + +Next, open ``propeller/plugins/config.yaml`` and remove the `default-env-vars `_ (no need to replace them, the default behavior is sufficient). + +Now if you re-run ``kustomize build flyte > flyte_generated.yaml``, you should see that the configmaps have been updated. + +Run ``kubectl apply -f flyte_generated.yaml`` to deploy these changes to your cluster for a production-ready deployment. diff --git a/rsts/conf.py b/rsts/conf.py new file mode 100644 index 0000000000..da93f8a9ea --- /dev/null +++ b/rsts/conf.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/stable/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) +import sphinx_rtd_theme + +# -- Project information ----------------------------------------------------- + +project = u'Flyte' +copyright = u'2019, Flyte Authors' +author = u'Flyte Authors' + +# The short X.Y version +version = u'' +# The full version, including alpha/beta/rc tags +release = u'0.0.1' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosectionlabel', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.extlinks', +] + +extlinks = { + 'propeller': ('https://github.com/lyft/flytepropeller/tree/master/%s', ''), + 'stdlib': ('https://github.com/lyft/flytestdlib/tree/master/%s', ''), + 'kit': ('https://github.com/lyft/flytekit/tree/master/%s', ''), + 'plugins': ('https://github.com/lyft/flyteplugins/tree/v0.1.4/%s', ''), + 'idl': ('https://github.com/lyft/flyteidl/tree/v0.14.1/%s', ''), + 'admin': ('https://github.com/lyft/flyteadmin/tree/master/%s', ''), +} + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Flytedoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'Flyte.tex', u'Flyte Documentation', + u'Flyte Authors', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'flyte', u'Flyte Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'Flyte', u'Flyte Documentation', + author, 'Flyte', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True diff --git a/rsts/contributor/components/admin.rst b/rsts/contributor/components/admin.rst new file mode 100644 index 0000000000..26a1bed016 --- /dev/null +++ b/rsts/contributor/components/admin.rst @@ -0,0 +1,343 @@ +.. _components-admin: + +########## +FlyteAdmin +########## + +Admin Structure +=============== + +FlyteAdmin serves the main Flyte API. It processes all client requests to the system. Clients include the FlyteConsole, which calls FlyteAdmin to list workflows, get execution details, etc., and FlyteKit, which calls FlyteAdmin to registering workflows, launch workflows etc. + +Below, we'll dive into each component defined in admin in more detail. + + +RPC +--- + +FlyteAdmin uses the `grpc-gateway `__ library to serve +incoming gRPC and HTTP requests with identical handlers. For a more detailed overview of the API, +including request and response entities, see the admin +service `definition `__. The +RPC handlers are a thin shim that enforce some request structure validation and call out to appropriate :ref:`manager `. +methods to process requests. + +.. _components-admin-manager: + +Managers +-------- + +The Admin API is broken up into entities: + +- Executions +- Launch plans +- Node Executions +- Projects (and their respective domains) +- Task Executions +- Tasks +- Workflows + +Each API entity has an entity manager in FlyteAdmin reposible for implementing business logic for the entity. +Entity managers handle full validation for create, update and get requests, and +data persistence in the backing store (see the :ref:`repository ` section). + + +Additional Components ++++++++++++++++++++++ + +The managers utilize additional components to process requests. These additional components include: + +- **:ref:`workflow engine `**: compiles workflows and launches workflow executions from launch plans. +- **:ref:`data ` (remote cloud storage)**: offloads data blobs to the configured cloud provider. +- **:ref:`runtime `**: loads values from a config file to assign task resources, initialization values, execution queues and more. +- **:ref:`async processes `**: provides functions for scheduling and executing workflows as well as enqueuing and triggering notifications + +.. _components-admin-repository: + +Repository +---------- +Serialized entities (tasks, workflows, launch plans) and executions (workflow-, node- and task-) are stored as protos defined +`here `__. +We use the excellent `gorm `__ library to interface with our database, which currently supports a postgres +implementation. The actual code for issuing queries with gorm can be found in the +`gormimpl `__ directory. + +Models +++++++ +Database models are defined in the `models `__ directory and correspond 1:1 with database tables [0]_. + +The full set of database tables includes: + +- executions +- execution_events +- launch_plans +- node_executions +- node_execution_events +- tasks +- task_executions +- workflows + +These database models inherit primary keys and indexes as defined in the corresponding `models `__ file. + +The repositories code also includes `transformers `__. +These convert entities from the database format to a response format for the external API. +If you change either of these structures, you will find you must change the corresponding transformers. + + +.. _components-admin-async: + +Component Details +================= + +This section dives into detail for each top-level directories defined in ``pkg/``. + +Asynchronous Components +----------------------- + +Notifications and schedules are handled by async routines that are reponsible for enqueing and subsequently processing dequeued messages. + +Flyteadmin uses the `gizmo toolkit `__ to abstract queueing implementation. Gizmo's +`pubsub `__ library offers implementations for Amazon SNS/SQS, Google's Pubsub, Kafka topics and publishing over HTTP. + +For the sandbox development, no-op implementations of the notifications and schedule handlers are used to remove external cloud dependencies. + + +Common +------ + +As the name implies, ``common`` houses shared components used across different flyteadmin components in a single, top-level directory to avoid cyclic dependencies. These components include execution naming and phase utils, query filter definitions, query sorting definitions, and named constants. + +.. _components-admin-data: + +Data +---- + +Data interfaces are primarily handled by the `storage `__ library implemented in flytestdlib. However, neither this nor the underlying `stow `__ library expose `HEAD `__ support so the data package in admin exists as the layer responsible for additional, remote data operations. + +Errors +------ + +The errors directory contains centrally defined errors that are designed for compatibility with gRPC statuses. + +.. _components-admin-config: + +Runtime +------- +Values specific to the flyteadmin application as well as task and workflow registration and execution are configured in the `runtime `__ directory. These interfaces expose values configured in the ``flyteadmin`` top-level key in the application config. + +.. _components-admin-workflowengine: + +Workflowengine +-------------- + +This directory contains interfaces to build and execute workflows leveraging flytepropeller compiler and client components. + + +Admin Service +============= + +Making Requests to FlyteAdmin +----------------------------- + +Adding request filters +++++++++++++++++++++++ + +We use `gRPC Gateway `_ to reverse proxy http requests into gRPC. +While this allows for a single implementation for both HTTP and gRPC, an important limitation is that fields mapped to the path pattern cannot be +repeated and must have a primitive (non-message) type. Unfortunately this means that repeated string filters cannot use a proper protobuf message. Instead use +the internal syntax shown below:: + + func(field,value) or func(field, value) + +For example, multiple filters would be appended to an http request:: + + ?filters=ne(version, TheWorst)+eq(workflow.name, workflow) + +Timestamp fields use the RFC3339Nano spec (ex: "2006-01-02T15:04:05.999999999Z07:00") + +The fully supported set of filter functions are + +- contains +- gt (greater than) +- gte (greter than or equal to) +- lt (less than) +- lte (less than or equal to) +- eq (equal) +- ne (not equal) +- value_in (for repeated sets of values) + +"value_in" is a special case where multiple values are passed to the filter expression. For example:: + + value_in(phase, 1;2;3) + +Filterable fields vary based on entity types: + +- Task + + - project + - domain + - name + - version + - created_at +- Workflow + + - project + - domain + - name + - version + - created_at +- Launch plans + + - project + - domain + - name + - version + - created_at + - updated_at + - workflows.{any workflow field above} (for example: workflow.domain) + - state (you must use the integer enum e.g. 1) + - States are defined in `launch_plan.proto `_. +- Executions (Workflow executions) + + - project + - domain + - name + - workflow.{any workflow field above} (for example: workflow.domain) + - launch_plan.{any launch plan field above} (for example: launch_plan.name) + - phase (you must use the upper-cased string name e.g. RUNNING) + - Phases are defined in `execution.proto `_. + - execution_created_at + - execution_updated_at + - duration (in seconds) + - mode (you must use the integer enum e.g. 1) + - Modes are defined in `execution.proto `_. + +- Node Executions + + - node_id + - execution.{any execution field above} (for example: execution.domain) + - phase (you must use the upper-cased string name e.g. QUEUED) + - Phases are defined in `execution.proto `_. + - started_at + - node_execution_created_at + - node_execution_updated_at + - duration (in seconds) +- Task Executions + + - retry_attempt + - task.{any task field above} (for example: task.version) + - execution.{any execution field above} (for example: execution.domain) + - node_execution.{any node execution field above} (for example: node_execution.phase) + - phase (you must use the upper-cased string name e.g. SUCCEEDED) + - Phases are defined in `execution.proto `_. + - started_at + - task_execution_created_at + - task_execution_updated_at + - duration (in seconds) + +Putting it all together +----------------------- + +If you wanted to do query on specific executions that were launched with a specific launch plan for a workflow with specific attributes, you could do something like: + +:: + + gte(duration, 100)+value_in(phase,RUNNING;SUCCEEDED;FAILED)+eq(lauch_plan.project, foo) + +eq(launch_plan.domain, bar)+eq(launch_plan.name, baz) + +eq(launch_plan.version, 1234) + +lte(workflow.created_at,2018-11-29T17:34:05.000000000Z07:00) + + + +Adding sorting to requests +++++++++++++++++++++++++++ + +Only a subset of fields are supported for sorting list queries. The explicit list is below: + +- ListTasks + + - project + - domain + - name + - version + - created_at +- ListTaskIds + + - project + - domain +- ListWorkflows + + - project + - domain + - name + - version + - created_at +- ListWorkflowIds + + - project + - domain +- ListLaunchPlans + + - project + - domain + - name + - version + - created_at + - updated_at + - state (you must use the integer enum e.g. 1) + - States are defined in `launch_plan.proto `_. +- ListWorkflowIds + + - project + - domain +- ListExecutions + + - project + - domain + - name + - phase (you must use the upper-cased string name e.g. RUNNING) + - Phases are defined in `execution.proto `_. + - execution_created_at + - execution_updated_at + - duration (in seconds) + - mode (you must use the integer enum e.g. 1) + - Modes are defined `execution.proto `_. +- ListNodeExecutions + + - node_id + - retry_attempt + - phase (you must use the upper-cased string name e.g. QUEUED) + - Phases are defined in `execution.proto `_. + - started_at + - node_execution_created_at + - node_execution_updated_at + - duration (in seconds) +- ListTaskExecutions + + - retry_attempt + - phase (you must use the upper-cased string name e.g. SUCCEEDED) + - Phases are defined in `execution.proto `_. + - started_at + - task_execution_created_at + - task_execution_updated_at + - duration (in seconds) + +Sorting syntax +-------------- + +Adding sorting to a request requires specifying the ``key``, e.g. the attribute you wish to sort on. Sorting can also optionally specify the direction (one of ``ASCENDING`` or ``DESCENDING``) where ``DESCENDING`` is the default. + +Example sorting http param: + +:: + + sort_by.key=created_at&sort_by.direction=DESCENDING + +Alternatively, since descending is the default, the above could be rewritten as + +:: + + sort_by.key=created_at + + +.. [0] Unfortunately, given unique naming constraints, some models are redefined in `migration_models `__ to guarantee unique index values. diff --git a/rsts/contributor/components/catalog.rst b/rsts/contributor/components/catalog.rst new file mode 100644 index 0000000000..9fc229c1a7 --- /dev/null +++ b/rsts/contributor/components/catalog.rst @@ -0,0 +1,61 @@ +.. _components-catalog: + +What is Data Catalog? +===================== + +Data Catalog (https://github.com/lyft/datacatalog) is a service for indexing parameterized, strongly-typed data artifacts across revisions. It allows for clients to query artifacts based on meta information and tags. + + +How Flyte Memoizes Task Executions on Data Catalog +-------------------------------------------------- + +Flyte :ref:`memoizes task executions<_concepts_catalog>` by creating artifacts in Data Catalog and associating meta information regarding the execution with the artifact. Let's walk through what happens when a task execution is cached on Data Catalog. + +Every task instance is represented as a DataSet: + +.. code-block:: JSON + + Dataset { + project: Flyte project the task was registered in + domain: Flyte domain for the task execution + name: flyte_task- + version: -- + } + +Every task execution is represented as an Artifact in the Dataset above: + +.. code-block:: JSON + + Artifact { + id: uuid + Metadata: [executionName, executionVersion] + ArtifactData: [List of ArtifactData] + } + + + ArtifactData { + Name: + value: + } + +To retrieve the Artifact, we tag the Artifact with a hash of the input values for the memoized task execution. + +.. code-block:: JSON + + ArtifactTag { + Name: flyte_cached- + } + +When caching an execution, Flyte propeller will: + +1. Create a dataset for the task +2. Create an artifact that represents the execution, along with the artifact data that represents the execution output +3. Tag the artifact with a unique hash of the input values + +When checking to see if the task execution is memoized Flyte Propeller will: + +1. Compute the tag by computing the hash of the input +2. Check if a tagged artifact exists with that hash + + a. If it does we have a cache hit and Propeller can skip the task execution. + b. If an artifact is not associated with the tag, Flyte Propeller needs to run the task. diff --git a/rsts/contributor/components/console.rst b/rsts/contributor/components/console.rst new file mode 100644 index 0000000000..97d6112a16 --- /dev/null +++ b/rsts/contributor/components/console.rst @@ -0,0 +1,124 @@ +.. _components-console: + +############# +Flyte Console +############# + +This is the web UI for the Flyte platform. + +********************* +Running flyteconsole +********************* + +===================== +Install Dependencies +===================== +Running flyteconsole locally requires `NodeJS `_ and +`yarn `_. Once these are installed, all of the dependencies +can be installed by running ``yarn`` in the project directory. + +====================== +Environment variables +====================== +Before we can run the server, we need to set up an environment variable or two. + +``ADMIN_API_URL`` (default: `window.location.origin `_) + +The Flyte console displays information fetched from the Flyte Admin API. This +environment variable specifies the host prefix used in constructing API requests. + +*Note*: this is only the host portion of the API endpoint, consisting of the +protocol, domain, and port (if not using the standard 80/443). + +This value will be combined with a suffix (such as ``/api/v1``) to construct the +final URL used in an API request. + +*Default Behavior* + +In most cases, ``flyteconsole`` will be hosted in the same cluster as the Admin +API, meaning that the domain used to access the console is the same value used to +access the API. For this reason, if no value is set for ``ADMIN_API_URL``, the +default behavior is to use the value of `window.location.origin`. + + +``BASE_URL`` (default: ``undefined``) + +This allows running the console at a prefix on the target host. This is +necessary when hosting the API and console on the same domain (with prefixes of +``/api/v1`` and ``/console`` for example). For local development, this is +usually not needed, so the default behavior is to run without a prefix. + + +``CORS_PROXY_PREFIX`` (default: ``/cors_proxy``) + +Sets the local endpoint for `CORS request proxying `_. + +=============== +Run the server +=============== + +To start the local development server, run ``yarn start``. This will spin up a +Webpack development server, compile all of the code into bundles, and start the +NodeJS server on the default port (3000). All requests to the NodeJS server will +be stalled until the bundles have finished. The application will be accessible +at http://localhost:3000 (if using the default port). + +************ +Development +************ + +========== +Storybook +========== + +This project has support for `Storybook `_. +Component stories live next to the components they test, in a ``__stories__`` +directory, with the filename pattern ``{Component}.stories.tsx``. + +You can run storybook with ``npm run storybook``, and view the stories at http://localhost:9001. + +============================= +Protobuf and the Network tab +============================= + +Communication with the Flyte Admin API is done using Protobuf as the +request/response format. Protobuf is a binary format, which means looking at +responses in the Network tab won't be very helpful. To make debugging easier, +each network request is logged to the console with it's URL followed by the +decoded Protobuf payload. You must have debug output enabled (on by default in +development) to see these messages. + +============ +Debug Output +============ + +This application makes use of the `debug `_ +libary to provide namespaced debug output in the browser console. In +development, all debug output is enabled. For other environments, the debug +output must be enabled manually. You can do this by setting a flag in +localStorage using the console: ``localStorage.debug = 'flyte:*'``. Each module in +the application sets its own namespace. So if you'd like to only view output for +a single module, you can specify that one specifically +(ex. ``localStorage.debug = 'flyte:adminEntity'`` to only see decoded Flyte +Admin API requests). + +.. _cors-proxy: + +============== +CORS Proxying +============== + +In the common hosting arrangement, all API requests will be to the same origin +serving the client application, making CORS unnecessary. For any requests which +do not share the same ``origin`` value, the client application will route +requests through a special endpoint on the NodeJS server. One example would be +hosting the Admin API on a different domain than the console. Another example is +when fetching execution data from external storage such as S3. This is done to +minimize the amount of extra configuration required for ingress to the Admin API +and data storage, as well as to simplify local development of the console without +the need to grant CORS access to ``localhost``. + +The requests and responses are piped through the NodeJS server with minimal +overhead. However, it is still recommended to host the Admin API and console on +the same domain to prevent unnecessary load on the NodeJS server and extra +latency on API requests due to the additional hop. diff --git a/rsts/contributor/components/index.rst b/rsts/contributor/components/index.rst new file mode 100644 index 0000000000..6b6014e600 --- /dev/null +++ b/rsts/contributor/components/index.rst @@ -0,0 +1,13 @@ +Flyte System Components +======================= + +These sections provide implementation details about specific flyte components. This information will be useful when contributing to a specific component. + +.. toctree:: + :maxdepth: 2 + :caption: Flyte Components + :name: componentstoc + + admin + catalog + console diff --git a/rsts/contributor/docs/index.rst b/rsts/contributor/docs/index.rst new file mode 100644 index 0000000000..0c8a0513cc --- /dev/null +++ b/rsts/contributor/docs/index.rst @@ -0,0 +1,36 @@ +.. _contribute-docs: + +###################### +Contributing to Docs +###################### + +Documentation for Flyte spans a few repos, and is hosted on GitHub Pages. The contents of the ``docs/`` folder on the ``master`` branch are hosted on GitHub Pages. See `GH Pages documentation `_ for more information. + +Documentation that the steps below will compile come from: + +* This repository, comprising written RST pages +* RST files generated from the Flyte IDL repository +* RST files generated from the Flytekit Python SDK + +Building +********** + +In order to create this set of documentation run:: + + $ make update_ref_docs && make all_docs + +What happens is: +* ``./generate_docs.sh`` runs. All this does is create a temp directory and clone the two aforementioned repos. +* The Sphinx builder container will run with files from all three repos (the two cloned one and this one) mounted. + + * It will generate RST files for Flytekit from the Python source code + * Copy RST files from all three repos into a common location + * Build HTML files into the ``docs/`` folder + +Please then commit the newly generated files before merging your PR. In the future we will invest in CI to help with this. + +Notes +******* +We aggregate the doc sources in a single ``index.rst`` file so that ``flytekit`` and the Flyte IDL HTML pages to be together in the same index/table of contents. + +There will be no separate Flyte Admin, Propeller, or Plugins documentation generation. This is because we have a :ref:`contributor` guide and high level usage/architecture/runbook documentation should either there, or into the administrator's guide. diff --git a/rsts/contributor/extending/index.rst b/rsts/contributor/extending/index.rst new file mode 100644 index 0000000000..f37d525378 --- /dev/null +++ b/rsts/contributor/extending/index.rst @@ -0,0 +1,81 @@ +.. _contributor-extending: + +Extending Flyte +=============== + +.. toctree:: + :maxdepth: 2 + :name: extendingtoc + :caption: Extending and customizing Flyte + + +Custom Tasks +############## +Writing logic for your own task types is the most common way to extend Flyte. In fact, Flyte ships with several extensions by default. These are tasks like the Qubole-run Hive queries or K8s-run Spark tasks, which were critical to Lyft's internal deployment of Flyte but aren't part of Flyte's core. + +Extending the IDL +******************** +Writing your own task will likely start with adding your own IDL message, which will look something like this :idl:`protos/flyteidl/plugins/sidecar.proto` or :idl:`protos/flyteidl/plugins/qubole.proto`. Your custom task's proto message can reference other objects like in the Qubole case, but ultimately it has to be one message class. + +An instance of this message class will be included alongside the rest of the ``TaskTemplate`` (in the ``custom`` field) for a given task. Your message should include all the additional information that the execution layer of Flyte will need to execute the task. That is, you don't need to worry about the container image, cpu/memory resources, input/output types, etc. since that is all covered in the normal task definition. You only need to worry about the custom information for your task. Technically, if your custom task doesn't need any additional information whatsoever, you can skip this step. + + +Extending the SDK +******************** +The next step is to write a task handler on the SDK side. That is, now that we have the definition of what your custom task will need, we need a way for users to write that task in Python, and then transform those tasks into task specifications containing that Protobuf message. Continuing with the above examples, we can look at how the SDK bits are built for the sidecar task and the Qubole Hive task. + +Broadly, the steps are: + +#. Define a task type. Concretely, this is just the string here :idl:`protos/flyteidl/core/tasks.proto#L92`. As mentioned above, technically you do not need an IDL if for some reason your custom task has no additional information whatsoever. But even in that case, you'll need a new task type string here. This is the key that the execution plane will reference to decide how to run your task. +#. Create a class for your custom task that wraps the base task class (``flytekit.common.tasks.sdk_runnable.SdkRunnableTask`` or just ``flytekit.common.tasks.task.SdkTask`` if a decorator is not required). +#. Optionally, create a decorator to make it simple to define user code. + +Qubole Hive Example +--------------------- +The Hive task is slightly more complicated in that it produces a futures file, but the basic steps are the same. + +#. First write a class that subclasses either the ``SdkRunnableTask`` or ``SdkTask`` like so :kit:`flytekit/common/tasks/hive_task.py#L27` +#. Override the :kit:`execute method ` to have the behavior that you want to see. In this case, we're running the user code, and then compiling the output of the user's code into our futures file. + * Also, an instance of the custom Protobuf message defined in the IDL should be created (if you need it), and added here. +#. Create a decorator for your task like this :kit:`flytekit/sdk/tasks.py#L623`. + +Ultimately, FlyteKit is a large wrapper around the Flyte IDL objects. That is, its primary function is to translate user Python code into the Flyte component protobufs, sometimes in multiple stages like in the Hive example, that the Flyte engine and control plane can then understand. + + +Extending Propeller +******************** + +Flyte plugins extend the Flyte execution plane with custom behavior for special tasks. When Flyte Propeller begins to run a task, it will look at the task type that you've defined and invoke the appropriate plugin. Spark tasks and Qubole Hive tasks are examples of tasks that are run by plugins. + +Structure of Plugins +----------------------- + +At a high level, a Flyte Propeller plugin is something that satisfies the ``Executor`` interface specified in :plugins:`go/tasks/v1/types/task.go`. The plugin's initialization code should register itself against the aforementioned task type string. When Propeller comes across a task with this type, the plugin will be invoked. Be cognizant that a plugin runs as a singleton in the engine. + +One of the important objects to understand is the ``TaskContext``. :plugins:`This interface ` will be an object created by Propeller, and supplied to your plugin code. Most importantly, the ``GetCustomState()`` function returns a custom struct that is integral to the cycle of your task's execution. + +You supply the initial value of this custom struct as the output of your ``StartTask`` call. On each call of the check loop thereafter, you get the version of the custom state that you returned before. Since this is the only state that is stored in a durable store (etcd), it should be your source of truth. + +.. note:: + Keep in mind that Flyte Propeller can restart at any time, which means your plugin can restart at any time. This custom state is the only state that your plugin can rely on. + +Note that this custom state is different than the custom IDL object that you previously defined. The IDL message should be thought of as data describing the task itself whereas this customstate should be thought of as a way to keep track of state during execution of your task. + +.. note:: + Note that while the ``CustomState`` object returned by the ``GetCustomState()`` function is a ``map[string]interface{}``, those interface values are not directly convertible to your Golang custom state objects. That is, they need to be first marshaled from JSON into bytes, and then unmarshaled from bytes back into your object, like so: :plugins:`go/tasks/v1/qubole/qubole_work.go#L187`. + +Task Initialization +^^^^^^^^^^^^^^^^^^^^ +The ``StartTask`` function is only called once and will be called with the task template containing the custom IDL struct if you chose to create one. You are only given this task template on this one ``StartTask`` call, so be sure your plugin code retrieves all the information from it that's necessary to complete the task's execution. For the Qubole Hive plugin for example, the queries to be run are copied from the custom IDL object into the custom state object. + +Task Updates +^^^^^^^^^^^^^^ +This is the function that will be called periodically by Flyte Propeller, and is responsible for deciding how your custom task is progressing. Note that while the task template is there in the function signature, it is not actually used and will always be nil. This was an unfortunate optimization that had to be made to save on S3 access times. + +Please refer to the generated documentation for a brief discussion of the other methods of the interface. + + +Background Updates +^^^^^^^^^^^^^^^^^^^ +Often you'll need something to monitor the progress of your execution. To this end, some plugins (like our Qubole Hive plugin and the waitable task plugin) make use of an `AutoRefreshCache `__, to which you can specify a sync function. This sync function will be periodically run on all objects in your cache. See the cache's documentation for more information. + diff --git a/rsts/contributor/index.rst b/rsts/contributor/index.rst new file mode 100644 index 0000000000..5c8e7a478a --- /dev/null +++ b/rsts/contributor/index.rst @@ -0,0 +1,18 @@ +.. _contributor: + +Contributor Docs +================ + +Before reading the contributor docs, it's best to read the :ref:`administrator` docs to understand how Flyte's major components fit together. + +The contributor provide low level implimentation details about how each Flyte component is architected, and explain how to extend the system with custom behavior. + +.. toctree:: + :maxdepth: 2 + :caption: Contributing Guidelines + :name: contribdocs + + components/index + language/index + extending/index + docs/index diff --git a/rsts/contributor/language/index.rst b/rsts/contributor/language/index.rst new file mode 100644 index 0000000000..b16c03de28 --- /dev/null +++ b/rsts/contributor/language/index.rst @@ -0,0 +1,200 @@ +.. _user-language: + +Flyte Specification Language +============================ + +Flyte at the core consists of a specification language that all the components of the system can interact with. +The specification language is written in `Protocol Buffers`_. This allows for a very portable language model. An implementation of the language +makes interaction with the system easier and brings the power of Flyte to the user. A full implementation of the language is provided in +:ref:`Python SDK ` as well as generated :ref:`Golang types `, C++ and JAVA. + +This is a reference manual for Flyte Specification Language. For more information on how to use the provided SDK, please +refer to :ref:`SDK manual `. + +Elements +-------- +.. _language-types: + +Types + Flyte Spec Language defines a flexible typing system. There are broadly three sets of types; primitives (e.g. ints, float, bool, string), + collections (lists and maps) and use-case specific types (e.g. Schema, Blob, Struct). + + Collection types support any dimensionality (e.g. a list of map of list of strings). + + .. note:: + Flyte collection types are invariant. i.e. a list of Cat is not a subtype of list of Animal. + + For reference documentation on types, refer to :ref:`Types ` + +.. _language-literals: + +Literals + Literals are bound values for specific types. E.g. `5` is a literal of primitive type int with value `5` + + For reference documentation on literals, refer to :ref:`Types ` + +.. _language-variables: + +Variables + Variables are named entities that have a type. Variables are used to name inputs and outputs of execution units (tasks, wfs... etc.) + as well as referencing outputs of other execution units. + + For reference documentation on variables, refer to :ref:`Variable ` + +.. _language-interface: + +Interface + Every execution unit in Flyte can optionally define an interface. The interface holds information about the inputs + and output variables. In order for **task B** to consume output **X** of **task A**, it has to declare an input + variable of the same type (not necessarily the same name). + + For reference documentation on interfaces, refer to :ref:`Interfaces ` + +.. _language-tasks: + +Tasks + TaskTemplate is a system entity that describes common information about the task for the rest of the system to reason + about. TaskTemplates contain: + + - An :ref:`identifier ` that globally identifies this task and allows other entities to reference + it. + - A string type that is used throughout the system to customize the behavior of the task execution (e.g. launch a + container in K8s, execute a SQL query... etc.) as well as how it gets rendered in the UI. + - An :ref:`interface ` that can be used to type check bindings as well as compatibility of plugging + the task in a larger workflow. + - An optional container that describes the versioned container image associated with the task. + - An optional custom Struct_ used by various plugins to carry information used to customize this task's behavior. + + For concept documentation on tasks, refer to :ref:`task concepts `. + For reference documentation on tasks, refer to :ref:`Tasks ` + +.. _language-bindings: + +Bindings + Bindings define how a certain input of a task should receive its :ref:`literal ` value. + Bindings can be static (assigned at compile/registration time to a literal) or can be references to outputs of other + nodes. + + A Binding of an input of type collection of strings can either be bound to an output of the same type of a different + node or else individual items can be bound to outputs of type string of other tasks. + + e.g. + .. code-block:: + + let taskA return output S of type String + let taskB return output SList of type list of string + let taskC takes input i of type list of string + + // Bind the entire input to the entire output. + taskC.i <- taskB.SList + + // Bind individual items + taskC.i[0] <- taskA.S + taskC.i[1] <- taskA.S + + For reference documentation on bindings, refer to :ref:`Types ` + +.. _language-identifiers: + +Identifiers + An identifier is a globally unique structure that identifies an entity in the system. Varrious entities have different + identifiers. Tasks, Workflows and Launchplans are all globally identified with an identifier. + + For reference documentation on identifiers, refer to :ref:`Indentifier ` + +.. _language-conditions: + +Conditional Expressions + Flyte Spec Language supports conditional expressions of two types; logical and comparison expressions. + Expressions are represented as a tree of boolean expressions. + + Logical Expressions + A logical expression is expressed as a logical operator (only AND and OR operators are supported) and two conditional + expressions + + Comparison Expressions + A comparison expression is expressed as a comparison operator + (Equal, Not Equal, Greater Than, Greater Than or Equal, Less Than or Less Than or Equal) and two operands. + + An operand can either be a :ref:`primitive ` or a variable name that exists as an input to the + node where the expression is evaluated. + + For reference documentation on conditions, refer to :ref:`Indentifier ` + +.. _language-nodes: + +Nodes + Nodes are encapsulations around any execution unit (task, workflow, launchplan... etc.) that abstracts away details about + the execution unit and allows interaction with other nodes with minimal information. + + Nodes define :ref:`how to bind ` the inputs of the underlying execution unit (e.g. task). + They can also alter how outputs are exposed by providing output aliases to some or all of the outputs variables. + + Multiple nodes can reference the same execution unit with different input bindings. + + Dependencies between nodes is driven by bindings; e.g. NodeC depends on data from NodeA, therefore the system will + wait to execute NodeC until NodeA has successfully finished executing. + Nodes can optionally define execution dependencies that are not expressed in bindings. + + For more information about the different types of nodes, please refer to :ref:`node concepts `. + For reference documentation on nodes, refer to :ref:`Nodes ` + +.. _language-workflows: + +Workflows + Workflows represent an execution scope in the system. A workflow is a directed-acyclic-graph that describes what steps + need to be executed, in what order as well as which steps need to consume data from other steps. + + For concept documentation on workflows, refer to :ref:`node concepts `. + For reference documentation on workflows, refer to :ref:`Workflows ` + +Compiler +-------- +Flyte system requires compiled Workflows to execute. A CLI is provided for convenience to compile locally. Flyte +automatically compiles all registered tasks and workflows. +The compilation process validates that all nodes and tasks interfaces match up. It then pre-computes the actual +execution plan and serializes the compiled workflow. + +Properties of types and values +------------------------------ + +Type Identity + Two types are equal if and only if their `Protocol Buffers`_ representation is identical. + +Assignability + Flyte types are invariant. Two variables can be assigned to each other if and only if their types + are identical. + +Extensibility +------------- + +New types + Flyte types can be extended in two ways: + + A customization of an existing type + e.g. Creating a URL type can be represented as a Literal Type String (that contains the final URL). As far as the task + interface is concerned, its output is of type String. A subsequent task can then have an input of type string to bind + to that output. + + Because this approach localizes the visibility of the new type to the one task that produced it, it's hard to enforce + any type checks (or validation for URL format... etc.) at the system layer. + + This approach won't always satsify more complex types' needs. + + Using generic types + Flyte offers a literal type `STRUCT` that allows the passing of any valid Struct_ as a value. + This approach is very powerful to pass other `Protocol Buffers`_ messages or custom types (represented in JSON_) + that only :ref:`plugins ` knows about. + + .. note:: + Flyte doesn't yet support subtyping, or custom strongly typed structs. + + +.. toctree:: + :maxdepth: 2 + :name: languagetoc + :caption: Spec Language + +.. _Protocol Buffers: https://en.wikipedia.org/wiki/Protocol_Buffers +.. _Struct: https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/struct.proto +.. _JSON: https://en.wikipedia.org/wiki/JSON \ No newline at end of file diff --git a/rsts/index.rst b/rsts/index.rst new file mode 100644 index 0000000000..63816991b6 --- /dev/null +++ b/rsts/index.rst @@ -0,0 +1,23 @@ +Flyte +===== + +About Flyte + +This is the public version of Flyte docs + +.. toctree:: + :maxdepth: 2 + :caption: Flyte Documentation + :name: mastertoc + + introduction/index + user/index + administrator/index + contributor/index + +.. toctree:: + :caption: Generated Documentation from Source + :maxdepth: 1 + :name: sourcecodedocstoc + + Flyte Language Specification diff --git a/rsts/introduction/docs_overview.rst b/rsts/introduction/docs_overview.rst new file mode 100644 index 0000000000..b128dbc42a --- /dev/null +++ b/rsts/introduction/docs_overview.rst @@ -0,0 +1,47 @@ +.. _introduction-docs: + +####################### +How to read these docs? +####################### + +Flyte can be interacted with and operated in 3 ways, or by 3 personas: +**user**, **administrator**, and **contributor**. The documentation is +organized in a way so that the reading flow is optimized for each of these +personas. + +User: I want to write Flyte Workflows +===================================== +A user refers to anyone who wants to: + + - Explore how Flyte works and try it out before installing, operating, or + using it + - Use a hosted Flyte deployment available at her organization + - Deploy a local cluster for debugging or testing + +:ref:`Jump to User Docs ` + + +Administrator: I want to manage a Flyte installation at my company +================================================================== +An administrator is someone who wants to deploy, manage, and scale a Flyte installation for +his or her organization. The administrator is not interested in altering or modifying any code, +only using the system off the shelf, and configuring the various available knobs and settings. This +section also talks about the typical installation structure and scalability +primitives available in the system. + +:ref:`Jump to Administrator Docs ` + + +Contributor: I want to help make Flyte even more awesome! +========================================================= +A contributor is someone who wants to: + - Extend Flyte's capabilities by contributing new plugins + - Fix a bug in the source code + - Gain a deep understanding of Flyte's architecture + - Add new functionality or alter the source code for their organization + - Contribute to this documentation + +We view Flyte enthusiasts as potential contributors and recommend they read +through these docs as well. + +:ref:`Jump to Contributor Docs ` diff --git a/rsts/introduction/index.rst b/rsts/introduction/index.rst new file mode 100644 index 0000000000..f60aa30c8a --- /dev/null +++ b/rsts/introduction/index.rst @@ -0,0 +1,14 @@ +.. _introduction: + +Introduction +============ + +Flyte description + +.. toctree:: + :maxdepth: 2 + :caption: Flyte Introduction + :name: introductiontoc + + whatis + docs_overview diff --git a/rsts/introduction/whatis.rst b/rsts/introduction/whatis.rst new file mode 100644 index 0000000000..ce6d813e67 --- /dev/null +++ b/rsts/introduction/whatis.rst @@ -0,0 +1,34 @@ +.. _introduction-what-is: + +############### +What is Flyte? +############### + +Workflows are ideal for organizing complex processes. As Lyft has evolved, developers have turned to workflows to solve an increasing number of business problems. Flyte was created as an end-to-end solution for developing, executing, and monitoring workflows reliably at scale. + +The system powers products all over Lyft, including Data Science, Pricing, Fraud, Driver Engagement, Locations, ETA, and Autonomous. + +Flyte handles all the overhead involved in executing complex workflows, including hardware provisioning, scheduling, data storage, and monitoring. This allows developers to focus solely on writing their workflow logic. + +Development +----------- + +In Flyte, workflows are expressed as a graph. To create a workflow, you must compose a graph that conforms to our strict specification. We provide developer tools that make it easy to define robust workflows that conform to this spec. The spec allows us to perform up-front validation and ensure data is passed properly from one task to the next. This practice has proven extremely useful at Lyft in time and cost savings. + +In as few as 20 lines of code, a user can define a custom workflow, submit the graph to Flyte, and snooze while Flyte fulfills the execution on a schedule, or on-demand. + +Execution +--------- + +When fulfilling a workflow execution, Flyte will dynamically provision the resources necessary to complete the work. Tasks are launched using docker containers to ensure consistency across environments. The system can distribute workloads across 100s of machines, making it useful for production model training and batch compute at Lyft. + +As tasks complete, unused resources will be deprovisioned to control costs. + +Workflows come with a variety of knobs such as automatic retries, and custom notifications on success or failure. + +Monitoring +---------- + +The Flyte system continuously monitors workflow executions throughout their lifecycle. You can visit the Flyte web interface at any time to track the status of a workflow. + +The web interface provides the ability to drill down into the individual workflow steps, and get rich error messaging to triage any issues. diff --git a/rsts/user/concepts/domains.rst b/rsts/user/concepts/domains.rst new file mode 100644 index 0000000000..d1ae8ddf2c --- /dev/null +++ b/rsts/user/concepts/domains.rst @@ -0,0 +1,11 @@ +.. _concepts-domains: + +Domains +======= + +Domains provide an abstraction to isolate resources and feature configuration for different +deployment environments. For example at Lyft we develop and deploy Flyte workflows in +development, staging, and production. We configure Flyte domains with those names, and specify +lower resource limits on the development and staging domains than production domains. We also +use Domains to disable Launchplans and schedules from development and staging domains, since +those features are typically meant for production deployments. diff --git a/rsts/user/concepts/dynamic_spec.rst b/rsts/user/concepts/dynamic_spec.rst new file mode 100644 index 0000000000..084fffbda9 --- /dev/null +++ b/rsts/user/concepts/dynamic_spec.rst @@ -0,0 +1,41 @@ +.. _concepts-dynamic-spec: + +Dynamic Job Spec +================ + +A dynamic job spec is a subset of the full workflow spec that defines a set of tasks, workflows as well as +nodes and output bindindgs that control how the job should assemble its outputs. + +This spec is currently only supported as an intermediate step in running :ref:`Dynamic Tasks `. + +.. code-block:: protobuf + :caption: Dynamic job spec in Protobuf + + message DynamicJobSpec { + repeated Node nodes = 1; + int64 min_successes = 2; + repeated Binding outputs = 3; + + repeated TaskTemplate tasks = 4; + repeated WorkflowTemplate subworkflows = 5; + } + +Tasks +----- + +Defines one or more :ref:`Tasks ` that can then be referenced in the spec. + +Subworkflows +------------ + +Defines zero or more :ref:`Workflows ` that can then be referenced in the spec. + +Nodes +----- + +Defines one or more :ref:`Nodes ` that can run in parallel to produce the final outputs of the spec. + +Outputs +------- + +Defines one or more binding that instructs engine on how to assemble the final outputs. \ No newline at end of file diff --git a/rsts/user/concepts/execution_timeline.rst b/rsts/user/concepts/execution_timeline.rst new file mode 100644 index 0000000000..48688ef6d3 --- /dev/null +++ b/rsts/user/concepts/execution_timeline.rst @@ -0,0 +1,7 @@ +.. _concepts-execution-timeline: + +######################################## +Timeline of a workflow execution +######################################## + +.. image:: https://raw.githubusercontent.com/lyft/flyte/assets/img/flyte_wf_timeline.svg?sanitize=true diff --git a/rsts/user/concepts/executions.rst b/rsts/user/concepts/executions.rst new file mode 100644 index 0000000000..f00b0b46b8 --- /dev/null +++ b/rsts/user/concepts/executions.rst @@ -0,0 +1,16 @@ +.. _concepts-executions: + +######################################## +Overview of the Execution of a Workflow +######################################## + +.. image:: https://raw.githubusercontent.com/lyft/flyte/assets/img/flyte_wf_execution_overview.svg?sanitize=true + +Typical flow using flyte-cli +----------------------------- + * When you request an execution of a Workflow using the UI, Flyte CLI or other stateless systems, the system first calls the + getLaunchPlan endpoint and retrieves a Launch Plan matching the name for a version. The Launch Plan definition includes the definitions of all the input variables declared for the Workflow. + * The user-side component then ensures that all required inputs are supplied and requests the FlyteAdmin service for an execution + * The Flyte Admin service validates the inputs, making sure that they are all specified and, if required, within the declared bounds. + * Flyte Admin then fetches the previously validated and compiled workflow closure and translates it to an executable format with all of the inputs. + * This executable Workflow is then launched on Kubernetes with an execution record in the database. diff --git a/rsts/user/concepts/index.rst b/rsts/user/concepts/index.rst new file mode 100644 index 0000000000..e72cae0985 --- /dev/null +++ b/rsts/user/concepts/index.rst @@ -0,0 +1,49 @@ +.. _concepts: + +Flyte Concepts +============== +This section provides a quick overview of some important concepts in Flyte with links +to deeper dives. + +:ref:`Tasks ` are at the core of Flyte. A Task is any independent unit of +processing. Tasks can be pure functions or functions with side-effects. Tasks also have +configuration and requirements specification associated with each definition of the task. + +:ref:`Workflows ` are programs that are guranteed to reach a terminal +state eventually. They are represented as Directed Acyclic Graphs (DAGs) expressed in protobuf. +The Flyte specification language expresses DAGs with branches, parallel steps and nested +Workflows. Workflow can optionally specify typed inputs and produce typed outputs, which +are captured by the framework. Workflows are composed of one or more +:ref:`Nodes `. A Node is an encapsulation of an instance of a Task. + +:ref:`Executions ` are instances of workflows, nodes or tasks created +in the system as a result of a user-requested execution or a scheduled execution. + +:ref:`Projects ` are a multi-tenancy primitive in Flyte that allow +logical grouping of Flyte workflows and tasks. Projects often correspond to source code +repositories. For example the project *Save Water* may include multiple `Workflows` +that analyze wastage of water etc. + +:ref:`Domains ` enable workflows to be executed in different environments, +with separate resource isolation and feature configuration. + +:ref:`Launchplans ` provide a mechanism to specialize input parameters +for workflows associated different schedules. + +:ref:`Registration ` is the process of uploading a workflow and its +task definitions to the :ref:`FlyteAdmin ` service. Registration creates +an inventory of available tasks, workflows and launchplans declared per project +and domain. A scheduled or on-demand execution can then be launched against one of +the registered entities. + +.. toctree:: + :maxdepth: 2 + :name: concepts + + tasks + workflows_nodes + executions + lanuchplans_schedules + projects + domains + registration diff --git a/rsts/user/concepts/launchplans_schedules.rst b/rsts/user/concepts/launchplans_schedules.rst new file mode 100644 index 0000000000..20c3ea1667 --- /dev/null +++ b/rsts/user/concepts/launchplans_schedules.rst @@ -0,0 +1,60 @@ +.. _concepts-launchplans: + +Launch plans +============ +Launch plans are used to execute workflows. A workflow can have many launch plans, but an individual launch plan is always tied to a single, specific workflow. Once created, launch plans are easy to share and execute. + +Launch plans provide a way to templatize Flyte workflow invocations. Launch plans contain a set of bound workflow inputs that are passed as arguments to create an execution. Launch plans do not necessarily contain the entire set of required workflow inputs, but a launch plan is always necessary to trigger an execution. Additional input arguments can be provided at execution time to supplement launch plan static input values. + +In addition to templatizing inputs, launch plans allow you to run your workflow on one or multiple schedules. Each launch plan can optionally define a single schedule (which can be easily disabled by disabling the launch plan) as well as optional notifications. Refer to :ref:`Notifications ` for a deep dive into available notifications. + +See `here `__ for an overview. + +What do launch plans provide? +----------------------------- + +- One click invocation of workflows with predefined inputs and friendly launch plan names. +- Multiple schedules with different default values for inputs per workflow. +- Ability to easily enable and disable schedules. +- Can be created dynamically with flyteclient or statically using the Flyte SDK. +- Associate different notifications with your workflows. +- Restrict inputs to be passed to the workflows at launch time using the fixed_inputs_. parameter. +- Multiple versions of the launch plan (with same name) with only one active version. Schedule will reflect only on the active launch plan version. + +.. _concepts-launchplans-inputs: + +Launch plan inputs +------------------ +Generally launch plan inputs correspond to their related workflow definition's inputs, in that the variable type and names are expected to match. Launch plans cannot introduce any inputs not defined in the core workflow definition. However, launch plan inputs differ slightly from workflow inputs in that the former are categorized into **default inputs** and **fixed inputs**. + +Default Inputs +^^^^^^^^^^^^^^ +Default inputs behave much like default workflow inputs. As their name implies, default inputs provide default workflow input values at execution time in the absence of any dynamically provided values. + +.. _fixed_inputs: + +Fixed Inputs +^^^^^^^^^^^^ +Fixed inputs cannot be overridden. If a workflow is executed with a launch plan and dynamic inputs that attempt to redefine the launch plan's fixed inputs, the execution creation request *will fail*. + +.. _concepts-schedules: + +Schedules +========= +Workflows can be run automatically using schedules associated with launch plans. Schedules can either define a cron_expression_. or rate_unit_. + +At most one launch plan version for a given {Project, Domain, Name} combination can be active, which means at most one schedule can be active for a launch plan. However, many unique launch plans and corresponding schedules can be defined for the same workflow. + +.. _cron_expression: + +Cron Expression +--------------- +Cron expression strings use the `AWS syntax `__. These are validated at launch plan registration time. + +.. _rate_unit: + +Rate Unit +--------- + +Schedules can also be defined using fixed rates in units of **days**, **hours** and **minutes**. + diff --git a/rsts/user/concepts/projects.rst b/rsts/user/concepts/projects.rst new file mode 100644 index 0000000000..c91d462202 --- /dev/null +++ b/rsts/user/concepts/projects.rst @@ -0,0 +1,7 @@ +.. _concepts-projects: + +Projects +======== +A project in Flyte is a grouping of workflows and tasks to achieve a particular goal. A Flyte project can map to an engineering project or everything that's owned by a team or an individual. There cannot be multiple projects with the same name in Flyte. Since the fully-qualified name for tasks and workflows include project name and domain name, the task/workflow names are only required to be unique within a project. The workflows in a project A can refer tasks and workflows from other projects using the fully-qualified name. + +Flyte allows users to set resource limit for each project. Flyte provides basic reports and dashboards automatically for each project. The information captured in these reports include workflow/task level insights, resource usage and billing information. diff --git a/rsts/user/concepts/registration.rst b/rsts/user/concepts/registration.rst new file mode 100644 index 0000000000..65e3af0254 --- /dev/null +++ b/rsts/user/concepts/registration.rst @@ -0,0 +1,35 @@ +.. _concepts-registrations: + +################################## +Understanding Registration process +################################## + +.. image:: https://raw.githubusercontent.com/lyft/flyte/assets/img/flyte_wf_registration_overview.svg?sanitize=true + +Typical Flow Using Flytekit +--------------------------- + + * A user defines tasks using the :ref:`Flytekit ` Task Definition language + * A User defines a workflow using either :ref:`Flytekit ` Workflow definition lanugage. + * The user then uses flytekit's register cli to compile the tasks into their serialized representation as described in :ref:`Flyte Specification language `. During this time the task representation is also bound to a container that contains the code for the task. This associated entity is registered with FlyteAdmin using the registerTask api. + * The user then uses flytekit's register cli to compile the workflow into their serialized representation as described in :ref:`Flyte Specification language `. The referenced tasks are replaced by their Flyte Admin registered Identifier, obtained in the previous step. This associated entity is registered with FlyteAdmin using the registerWorkflow api. + * She can then launch an execution using the FlyteAdmin launch execution API, which requires the necessary inputs provided. This is automatically done if she uses the Flyte-CLI to launch the + execution. + * She can later use the FlyteAdmin read API's to get details of the execution, + monitor it to completion or retrieve a historical execution + * OR alternatively she can use the FlyteConsole to visualize the execution in + realtime as it progresses or visualize any historical execution. The console + also makes it easy to view debugging information for the execution. + * She can set specific rules like *notification* on failure or success or + publish all events in an execution to a pub-sub system. + * She can also query the datastore to get a summary of all the executions and + the compute resources consumed. + +Typical Flow without Flytekit +------------------------------------ +It is possible to achieve the exact same workflow as above in case Flytekit is not available. Workflows and tasks are purely specifications and can be provided using any tool like YAML, JSON, protobuf binary or any other programming language. Contributions welcome. + +Registration in the Backend +--------------------------- + +When FlyteAdmin receives a workflow registration request, it uses the workflow compiler to compile and validate the workflow. It also fetches all the referenced tasks and creates a complete workflow closure, which is stored in the metastore. If the workflow compilation fails, the compiler returns an error to the client. diff --git a/rsts/user/concepts/tasks.rst b/rsts/user/concepts/tasks.rst new file mode 100644 index 0000000000..b6365c4398 --- /dev/null +++ b/rsts/user/concepts/tasks.rst @@ -0,0 +1,82 @@ +.. _concepts-tasks: + +Tasks +===== + +Definition +---------- + +Tasks encapsulate fully independent units of execution. Flyte language exposes an extensible model +to express that in an execution-independent language. Flyte contains first class task plugins that take +care of executing these tasks. + +Real world examples +------------------- + +Query a data store + Using :ref:`Hive tasks ` to retrieve data into dataframes so that subsequent tasks can process them. + +Transform data + Using :ref:`Container tasks ` to transform data collected/computed earlier, users can develop a task as + a simple Lambda function with specified inputs and outputs represented as a container with entrypoints, with specific compute, memory and + gpu requirements. + +Map-reduce massive jobs + Using :ref:`Spark programs ` with their cluster configuration and compute requirements + +Hyperparameter tuning task + Using a system like Katib, users can execute a task that needs multiple iterations and leads to multiple other containers to execute. + +A distributed or single container Tensorflow Job + +Characteristics +--------------- + +In abstract, a task in the system is characterized by: + +1. A :ref:`project and domain ` combination, +2. A unique unicode name (we recommend it not to exceed 32 characters), and +3. A version string. +4. *Optional* Task interface definition + + In order for tasks to exchange data with each other, a task can define a signature (much like a function/method + signature in programming languages). A task interface defines the :ref:`input and output variables ` + as well as their :ref:`types `. + +Requirements +------------ + +When deciding whether a unit of execution conistitutes a Flyte Task or not. Consider the following: + +- Is there a well-defined graceful/successful exit criteria for the task? A task is expected to exit after finishing processing + its inputs. + +- Is it repeatable? Under certain circumstances, a task might be retried, rerun... etc. with the same inputs. It's expected + to produce the same outputs every single time. For example, avoid using random number generators with current clock as seed + but opt to using a system-provided clock as the seed. + +- Is it a pure function? i.e. does it have side effects that are not known to the system (e.g. calls a web-service). It's strongly + advisable to avoid side-effects in tasks. When side-effects are required, ensure that those operations are idempotent_. + +Types +----- +Since it's impossible to define the unit of execution of a task the same way for all kinds of tasks, Flyte allows different task +types in the system. Flyte comes with a set of defined, battle tested task types but also allows for a very flexible model to +introducing new :ref:`user-defined task types `. Read more about various supported task types :ref:`here `. + +Fault tolerance +--------------- +In any distributed system failure is inevitable, allowing users to design a fault-tolerant system (e.g. workflow) is an inherent goal of Flyte. At a high level, tasks offer two parameters to control how to handle that: + +Retries + Tasks can define a retry strategy to let the system know how to handle failures (e.g. retry 3 times on any errors). + +Timeouts + In order for the system to ensure it's always making progress, tasks must be guaranteed to end. The system defines a default timeout + period for tasks. It's also possible for task authors to define a timeout period after which the task is marked as failure. Note that + a timed-out task will be retried if it has a retry strategy defined. + +Memoization +----------- +Flyte supports memoization for task outputs to ensure identical invocations of a task are not repeatedly executed wasting compute resources. +For more information on memoization please refer to :ref:`Task Cache `. diff --git a/rsts/user/concepts/workflows_nodes.rst b/rsts/user/concepts/workflows_nodes.rst new file mode 100644 index 0000000000..da1672879a --- /dev/null +++ b/rsts/user/concepts/workflows_nodes.rst @@ -0,0 +1,66 @@ +.. _concepts-workflows: + +Workflows +========= + +A workflow is a directed acyclic graph (DAG) of units of work encapsulated by nodes. Specific instantiations of a workflow (commonly with bound input arguments) +are referred to as workflow executions, or just executions. In other words, a workflow is a template for ordered task execution. + +Flyte workflows are defined in protobuf and the Flyte SDK facilitates writing workflows. Users can define workflows as a collection of nodes. +Nodes within a workflow can produce outputs that subsequent nodes depend on as inputs. These dependencies dictate the workflow structure. +Workflows written using the SDK do not need to explicitly define nodes to enclose execution units (tasks, sub-workflows, launch plans), these will be injected by the SDK and captured at registration time. + +Structure +--------- + +Workflows can accept inputs and/or produce outputs and re-use task definitions across :ref:`projects ` and :ref:`domains `. +Workflow structure is flexible - nodes can be executed in parallel, the same task definition can be re-used within the same workflow. A single workflow +can contain any combination of task types. A workflow can contain a single functional node, it can contain many nodes in all sorts of arrangements and even launch other workflows. + +At execution time, node executions will be triggered as soon as their inputs are available. Workflow nodes are naturally run in parallel when possible. + +Flyte-specific structure +^^^^^^^^^^^^^^^^^^^^^^^^ +During registration, the Flyte platform will validate workflow structure and save the workflow. The registration process also updates the workflow graph. +A compiled workflow will always have a start and end node injected into the workflow graph. In addition a failure handler will catch and process execution failures. + +Versioning +---------- +Like :ref:`tasks `, workflows are versioned. Registered workflows are also immutable meaning that an instance of a workflow defined +by project, domain, name and version cannot be updated. The tasks referenced in a workflow version are themselves immutable and are tied to specific tasks' versions. + +Executions +---------- + +A workflow can only be executed through a :ref:`launch plan `. +A workflow can be launched many times with a variety of launch plans and inputs. Workflows that produce inputs and outputs can take advantage of :ref:`task caching ` to cache intermediate inputs and outputs and speed-up subsequent executions. + +.. _concepts-nodes: + +Nodes +===== + +A node represents a unit of execution or work within a workflow. Ordinarily a node will encapsulate an instance of a :ref:`task ` but a node can also contain an entire subworkflow or trigger a child workflow. Nodes can have inputs and outputs, which are used to coordinate task inputs and outputs. Node outputs can be used as inputs to other nodes within a workflow. + +Tasks are always encapsulated within a node, however, like tasks, nodes can come in a variety of flavors determined by their *target*. +These targets include :ref:`task nodes `, :ref:`workflow nodes `, and :ref:`branch nodes `. + +.. _concepts-task-nodes: + +Task Nodes +---------- + +Tasks referenced in a workflow are always enclosed in nodes. This extends to all task types, for example an array task will be enclosed by a single node. For more information about different task types, refer to :ref:`task types `. + +.. _concepts-workflow-nodes: + +Workflow Nodes +-------------- +A node can contain an entire sub-workflow. Because workflow executions always require a launch plan, workflow nodes have a reference to a launch plan used +to trigger their enclosed workflows. + +.. _concepts-branch-nodes: + +Branch Nodes +------------ +Branch nodes alter the flow of the workflow graph. Conditions at runtime are evaluated to determine control flow. diff --git a/rsts/user/features/flytecli.rst b/rsts/user/features/flytecli.rst new file mode 100644 index 0000000000..5d6ce28cd9 --- /dev/null +++ b/rsts/user/features/flytecli.rst @@ -0,0 +1,151 @@ +.. _features-flytecli: + +======== +FlyteCLI +======== + +--------------------------------------------------- +A command-line interface for interacting with Flyte +--------------------------------------------------- + +The FlyteCLI is a command-line tool that allows users to perform administrative +tasks on their Flyte workflows and executions. It is an independent module but +installed as part of the `Flyte Kit `. It primarily +iteracts with the `FlyteAdmin ` service over its gRPC +interface, allowing users to list registered workflows, or get a currently +running execution. + +------ + +Installation +============ + +The easist way to install FlyteCLI is using virtual environments. +Follow the official doc_ to install the ``virtualenv`` package if +you don't already have it in your development environment. + +Install from source +------------------- +Now that you have virtualenv, you can either install flyte-cli from source. +To do this first clone the git repository and +after setting up and activating your virtual environment, change directory to +the root directory of the flytecli package, and install the dependencies with +``pip install -e .``. + + +.. _doc: https://virtualenv.pypa.io/en/latest/installation/ + +Install from pypi [recommended] +------------------------------- +Another option is to just install flyte-cli from prebuilt binaries + +Testing if you have a working installation +------------------------------------------ + +To test whether you have a successful installation of flytecli, run +``flyte-cli`` or ``flyte-cli --help``. + +If you see the following output, you have installed the FlyteCLI successfully. + +.. code-block:: console + + Usage: flyte-cli [OPTIONS] COMMAND [ARGS]... + + Command line tool for interacting with all entities on the Flyte Platform. + + Options: + -n, --name TEXT [Optional] The name to pass to the sub-command (if + ... + + Commands: + execute-launch-plan Kick off a launch plan. + ... + + +------ + +Terminology +=========== + +This section introduces and explains the most commonly used terms and concepts +the users will see in FlyteCLI. + +Host +---- +``Host`` refers to your running Flyte instance and is a common +argument for the commands in FlyteCLI. The FlyteCLI will only be interacting +with the Flyte instance at the URL you specify with the ``host`` argument. +parameter. This is a required argument for most of the FlyteCLI commands. + +Project +------- +``Project`` is a multi-tenancy primitive in Flyte and allows logical grouping +of instances of Flyte entities by users. Within Lyft's context, this term +usually refers to the name of the Github repository in which your workflow +code resides. + +For more information see :ref:`Projects ` + +Domain +------ +The term ``domain`` refers to development environment (or the service instance) +of your workflow/execution/launch plan/etc. You can specify it with the +``domain`` argument. Values can be either ``development``, ``staging``, or +``production``. See :ref:`Domains ` + + +Name +---- +The ``name`` of a named entity is a randomly generated hash assigned +automatically by the system at the creation time of the named entity. For some +commands, this is an optional argument. + + +Named Entity +------------ +``Name Entity`` is a primitive in Flyte that allows logical grouping of +processing entities across versions. The processing entities to which this term +can refer include unversioned ``launch plans``, ``workflows``, +``executions``, and ``tasks``. In other words, an unversioned ``workflow`` named +entity is essentially a group of multiple workflows that +have the same ``Project``, ``Domain``, and ``Name``, but different versions. + + +URN +--- + +.. note:: + + URN is a FlyteCLI-only concept. You won't see this term in other flyte modules. + +URN stands for "unique resource name", and is the identifier of +a version of a given named entity, such as a workflow, a launch plan, +an execution, or a task. Each URN uniquely identifies a named entity. +URNs are often used in FlyteCLI to interact with specific named entities. + +The URN of a version of a name entity is composible from the entity's +attributes. For example, the URN of a workflow can be composed of a prefix +`wf` and the workflow's ``project``, ``domain``, ``name``, and ``version``, +in the form of ``wf::::``. + +Note that execution is the sole exception here as an execution does not +have versions. The URN of an execution, therefore, is in the form of +``ex:::``. + +------ + +Getting help +============ + +FlyteCLI uses subcommands. Whenever you feel unsure about the usage or +the arguments of a command or a subcommand, get help by running +``flyte-cli --help`` or ``flyte-cli --help`` + + +------ + +Commands +======== + +For information on available commands in FlyteCLI, refer to FlyteCLI's +help message. diff --git a/rsts/user/features/index.rst b/rsts/user/features/index.rst new file mode 100644 index 0000000000..162eba13bf --- /dev/null +++ b/rsts/user/features/index.rst @@ -0,0 +1,16 @@ +.. _user-features: + +Flyte Features +============== + +.. toctree:: + :maxdepth: 2 + :name: features + + flytecli + observability + notifications + labels_annotations + launchplans + task_cache + roles diff --git a/rsts/user/features/labels_annotations.rst b/rsts/user/features/labels_annotations.rst new file mode 100644 index 0000000000..5043ad6866 --- /dev/null +++ b/rsts/user/features/labels_annotations.rst @@ -0,0 +1,53 @@ +.. _features-labels-annotations: + +Labels and Annotations +====================== +In Flyte, workflow executions are created as kubernetes resources. These can be extended with +`labels `_ and +`annotations `_. + +**Labels** and **annotations** are key value pairs which can be used to identify workflows for your own uses. +Labels are meant to be used as identifing attributes whereas annotations are arbitrary, *non-identifying* metadata. + +Using labels and annotations is entirely optional. They can be used to categorize and identify workflow executions. + +Labels and annotations are optional parameters to launch plan and execution invocations. In the case an execution +defines labels and/or annotations *and* the launch plan does as well, the execution spec values will be preferred. + +Launch plan usage example +------------------------- + +.. code:: python + + from flytekit.models.common import Labels, Annotations + + @workflow + class MyWorkflow(object): + ... + + my_launch_plan = MyWorkflow.create_launch_plan( + labels=Labels({"myexecutionlabel": "bar", ...}), + annotations=Annotations({"region": "SEA", ...}), + ... + ) + + my_launch_plan.execute(...) + +Execution example +----------------- + +.. code:: python + + from flytekit.models.common import Labels, Annotations + + @workflow + class MyWorkflow(object): + ... + + my_launch_plan = MyWorkflow.create_launch_plan(...) + + my_launch_plan.execute( + labels=Labels({"myexecutionlabel": "bar", ...}), + annotations=Annotations({"region": "SEA", ...}), + ... + ) diff --git a/rsts/user/features/lanuchplans.rst b/rsts/user/features/lanuchplans.rst new file mode 100644 index 0000000000..4f66861a6a --- /dev/null +++ b/rsts/user/features/lanuchplans.rst @@ -0,0 +1,131 @@ +.. _fetaures-lanuchplans: + +Flyte Launchplans +################# + +When to use launchplans? +======================== + +- I want multiple schedules for my workflow with zero or more predefined inputs. +- I want to run the same workflow but with a different set of notifications. +- I want to share my workflow with another user with the inputs already set so that the other user can simply kick off an execution. +- I want to share my workflow to another user but also make sure that some inputs can be overridden if needed. +- I want to share my workflow with another user but make sure that some inputs are never changed. + + +Partial Inputs for Launchplans +============================== +Launch plans bind a partial or complete list of inputs necessary to launch a workflow. Launch plan inputs must only assign inputs already defined in the reference workflow definition. Refer to to :ref:`primary launch plan concept documentation ` for a detailed introduction to launch plan input types. + +For example, let's say you had the following workflow definition: + +.. code:: python + + @workflow_class + class MyWorkflow(object): + sample_size = Input(Types.Integer, required=False, default=100) + region = Input(Types.String) + run_date = Input(Types.Datetime) + ... + +If you wanted to run the workflow over a set of date ranges for a specific region, you could define the following launch plan: + +.. code:: python + + sea_launch_plan = MyWorkflow.create_launch_plan( + default_inputs={'sample_size': 1000}, + fixed_inputs={'region': 'SEA'}, + ) + +Some things to note here, we redefine the ``sample_size`` input in the launch plan - this is perfectly fine. Workflow inputs with default values can be redefined in launch plans. If we decide at execution creation time to adjust ``sample_size`` once more that's also perfectly fine because default_inputs can be overriden. However the region input is *fixed* and cannot be overriden. + +The launch plan doesn't assign run_date but there is nothing wrong with creating a launch plan that assigns all workflow inputs (either as default or fixed inputs). Of course all required inputs must be resolved at execution time, but execution creation requests can always accept dynamic inputs should your launch plan not define the complete set. + +Let's take a look at how to use the launch plan to create executions programmatically: + +.. code:: python + + from datetime import timedelta, date + + run_date = date(2018, 1, 1) + end_date = date(2019, 1, 1) + one_day = timedelta(days=1) + while run_date <= end_date: + sea_launch_plan.execute('myflyteproject', 'myflytedomain', inputs={'run_date': run_date}) + run_date += one_day + +And boom, you've got a year's worth of executions created! + +Let's say for 2019 data you've realized it makes sense to change the sample size. All your other data is in place and you can't wait to be notified once this last workflow has completed. Easy! you can configure your launch plan like so: + +.. code:: python + + from datetime import timedelta, date + + improved_sea_launch_plan = MyWorkflow.create_launch_plan( + default_inputs={'sample_size': 2000}, + fixed_inputs={'region': 'SEA', 'run_date': date(2019, 1, 1)}, + notifications=[notifications.Email([WorkflowExecutionPhase.SUCCEEDED], ["me@myemail.com"])] + ) + improved_sea_launch_plan.execute('myflyteproject', 'myflytedomain', name="one_last_execution") + +Schedules for Launchplans +========================== + +Schedules can be expressed as :ref:`rate units or cron expressions `. + +For example, considering the following workflow definition with a triggered-at time input: + +.. code:: python + + @workflow_class + class MyWorkflow(object): + triggered_time = Input(Types.Datetime) + an_integer_input = Input(Types.Integer) + an_alternative_input = Input(Types.Integer, required=False, default=10) + ... + +To run ``MyScheduledWorkflow`` every 5 minutes with a value set for ``an_integer_input`` and the scheduled execution time assigned to the ``triggered_time`` input you could define the following launch plan: + +.. code:: python + + my_schedule = schedule.FixedRate(datetime.timedelta(minutes=5), kickoff_time_input_arg="triggered_time") + my_fixed_rate_launch_plan = MyWorkflow.create_launch_plan( + default_inputs={'an_integer_input': Input(Types.Integer, default=900)}, + schedule=my_schedule, + ) + +Workflows that run on a schedule do not always need a corresponding triggered at input value. +For example, take the following workflow: + +.. code:: python + + @workflow_class + class MyWorkflow(object): + an_integer_input = Input(Types.Integer) + an_alternative_input = Input(Types.Integer, required=False, default=10) + ... + + +The above can be run on a cron schedule every 5 minutes like so: + +.. code:: python + + my_cron_launch_plan = MyWorkflow.create_launch_plan( + default_inputs={'an_integer_input': Input(Types.Integer, default=5)}, + schedule=schedules.CronSchedule('0/5 * * * ? *'),, + ) + +Once you've initialized your launch plan, don't forget to set it to active so that the schedule is run. + +.. code:: python + + my_fixed_rate_launch_plan.update(LaunchPlanState.ACTIVE) + # Our fixed rate schedule is now active. + + my_cron_launch_plan.update(LaunchPlanState.ACTIVE) + # Our cron schedule is now active too. + + # Too many active, let's disable one! + my_fixed_rate_launch_plan.update(LaunchPlanState.INACTIVE) + diff --git a/rsts/user/features/notifications.rst b/rsts/user/features/notifications.rst new file mode 100644 index 0000000000..634e94d1bb --- /dev/null +++ b/rsts/user/features/notifications.rst @@ -0,0 +1,30 @@ +.. _concepts-notifications: + +Notifications +============= + +Notifications are customizable events triggered on workflow termination. + +Notifications can be of three flavors: + +- `Email <#howto-email-notifications>`__ +- `PagerDuty <#howto-pagerduty-notifications>`__ +- `Slack <#howto-slack>`__ + +You can combine notifications in a single workflow to trigger for different +combinations of recipients, domains and terminal phases. + +Terminal workflow phases include: + +- ``WorkflowExecution.Phase.FAILED`` + +- ``WorkflowExecution.Phase.TIMED_OUT`` + +- ``WorkflowExecution.Phase.ABORTED`` + +- ``WorkflowExecution.Phase.SUCCESS`` + + +Future work +----------- +Currently the notification email subject and body are hard-coded. In the future passing these as customizable parameters in the notification proto message would be helpful. diff --git a/rsts/user/features/observability.rst b/rsts/user/features/observability.rst new file mode 100644 index 0000000000..566e586e89 --- /dev/null +++ b/rsts/user/features/observability.rst @@ -0,0 +1,76 @@ +.. _features-observability: + +Metrics for your executions +=========================== + +Flyte-Provided Metrics +~~~~~~~~~~~~~~~~~~~~~~~ +Whenever you run a workflow, Flyte Platform automatically emits high-level metrics. These metrics follow a consistent schema and +aim to provide visibility into aspects of the Platform which might otherwise be opaque. These metrics will help users diagnose whether an issue is inherent to the Platform or to one's own task or workflow implementation. We will be adding to this set of metrics as we implement the capabilities to pull more data from the system, so keep checking back for new stats! + +At a highlevel, workflow execution goes through the following discrete steps: + +.. image:: https://raw.githubusercontent.com/lyft/flyte/assets/img/flyte_wf_timeline.svg?sanitize=true + +=================================== ================================================================================================================================== + Description of main events for workflow execution +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Events Description +=================================== ================================================================================================================================== +Acceptance Measures the time between when we receive service call to create an Execution (Unknown) and when it has moved to Queued. +Transition Latency Measures the latency between two consecutive node executions, the time spent in Flyte engine. +Queuing Latency Measures the latency between the time a node's been queued to the time the handler reported the executable moved to running state. +Task Execution Actual time spent executing user code +Repeat steps 2-4 for every task +Transition Latency See #2 +Completion Latency Measures the time between when the WF moved to succeeding/failing state and when it finally moved to a terminal state. +=================================== ================================================================================================================================== + + +========================================================== =========== =============================================================================================================================================================== + Flyte Stats Schema +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Prefix Type Description +========================================================== =========== =============================================================================================================================================================== +``propeller.all.workflow.acceptance-latency-ms`` Timer (ms) Measures the time between when we receive service call to create an Execution (Unknown) and when it has moved to Queued. +``propeller.all.node.queueing-latency-ms`` Timer (ms) Measures the latency between the time a node's been queued to the time the handler reported the executable moved to running state. +``propeller.all.node.transition-latency-ms`` Timer (ms) Measures the latency between two consecutive node executions, the time spent in Flyte engine. +``propeller.all.workflow.completion-latency-ms`` Timer (ms) Measures the time between when the WF moved to succeeding/failing state and when it finally moved to a terminal state. +``propeller.all.node.success-duration-ms`` Timer (ms) Actual time spent executing user code (when the node ends with success phase). +``propeller.all.node.success-duration-ms-count`` Counter Count of how many times a node success was reported. +``propeller.all.node.failure-duration-ms`` Timer (ms) Actual time spent executing user code (when the node ends with failure phase). +``propeller.all.node.failure-duration-ms-count`` Counter Count of how many times a node failure was reported. + +========================================================== =========== =============================================================================================================================================================== + +All the above stats are automatically tagged with the following fields for further scoping. This includes user-produced stats. Users +can also provide additional tags (or override tags) for custom stats. + +.. _task_stats_tags: + +=============== ================================================================================= + Flyte Stats Tags +-------------------------------------------------------------------------------------------------- + Tag Description +=============== ================================================================================= +wf This is the name of the workflow that was executing when this metric was emitted. + ``{{project}}:{{domain}}:{{workflow_name}}`` +=============== ================================================================================= + +Flyte Task Container Metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +To be built. + +Default Project Dashboard +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +To be built. + +User Stats With Flyte +~~~~~~~~~~~~~~~~~~~~~~ +The workflow parameters object that the SDK injects into the various tasks has a statsd handle that users should call +to emit stats related to their workflows not captured by the default metrics. Please see the +:ref:`tutorial ` for more information. The usual caveats around cardinality apply of course. + +Users are encouraged to avoid creating their own stats handlers. These can pollute the general namespace if not done +correctly, and also can accidentally interfere with production stats of live services, causing pages and wreaking +havoc in general. In fact, if you're using any libraries that emit stats, it's best to turn them off if possible. diff --git a/rsts/user/features/roles.rst b/rsts/user/features/roles.rst new file mode 100644 index 0000000000..7b5160fb1c --- /dev/null +++ b/rsts/user/features/roles.rst @@ -0,0 +1,43 @@ +.. _features-roles: + +Why roles? +========== +Roles can be applied to launch plans (and the executions they create) to determine execution privileges. +Out of the box, Flyte offers a way to configure permissions via IAM role or Kubernetes service account. + +IAM role examples +----------------- + +Configure project-wide IAM roles by adding the following to your config: + +.. code:: python + + [auth] + assumable_iam_role=arn:aws:iam::123accountid:role/my-super-cool-role + + +Alternatively, pass the role as an argument to ``create_launch_plan``: + +.. code:: python + + my_lp = MyWorkflow.create_launch_plan(assumable_iam_role='arn:aws:iam::123accountid:role/my-super-cool-role') + + +Kubernetes serviceaccount examples +---------------------------------- + +Configure project-wide kubernetes serviceaccounts by adding the following to your config: + +.. code:: python + + [auth] + kubernetes_service_account=my-kube-service-acct + + +Alternatively, pass the role as an argument to ``create_launch_plan``: + +.. code:: python + + my_lp = MyWorkflow.create_launch_plan(kubernetes_service_account='my-kube-service-acct') + + diff --git a/rsts/user/features/task_cache.rst b/rsts/user/features/task_cache.rst new file mode 100644 index 0000000000..7cd816f5ad --- /dev/null +++ b/rsts/user/features/task_cache.rst @@ -0,0 +1,34 @@ +.. _features-task_cache: + +Task Cache +========== + +Flyte provides the ability to cache the output of task executions in order to make subsequent executions faster. A well-behaved Flyte Task should generate deterministic output given the same inputs and task functionality. This is useful in situations where a user knows that many executions with the exact same inputs can occur. For example, your task may be periodically run on a schedule, run multiple times when debugging workflows, or commonly shared across different workflows but receive the same inputs. + +Enable Caching For a Task +------------------------- + +In order to enable your task to be cached, mark ``cache=True`` below: + +.. code-block:: python + + @inputs(original=Types.String) + @outputs(hash=Types.String) + @python_task(cache=True, cache_version='1.0.0') + def hash_string_task(wf_params, original, hash) + ... + +A task execution is cached based on the **Project, Domain, cache_version, the task signature and inputs** associated with the execution of the task. + +- *Project:* A task run under one project cannot use the cached task execution from another project. This could cause inadvertent results between project teams that could cause data corruption. +- *Domain:* For separation of test, staging, and production data, task executions are not shared across these environments. +- *cache_version:* When task functionality changes, you can change the cache_version of the task. Flyte will know not to use older cached task executions and create a new cache entry on the next execution. +- *Task signature:*: The cache is specific to the task signature that is associated with the execution. The signature is made up of task name, input parameter names/types and also the output parameter name/types. +- *Task input values*: A well-formed Flyte Task always produces deterministic outputs. This means given a set of input values, every execution should produce identical outputs. When a task execution is cached, the input values are part of the cache key. + +Notice that task executions can be cached across different versions of the task. This is because a change in SHA does not neccessarily mean that it correlates to a change in task functionality. + +Flyte provides several ways to break the old task execution cache, and cache new output: + +- ``cache_version``: this field indicates that the task functionality has changed. Flyte users can manually update this version and Flyte will cache the next execution instead of relying on the old cache. +- Task signature: If a Flyte user changes the task interface in any way (such as by adding, removing, or editing inputs/outputs), Flyte will treat that as a task functionality change. On the next execution, Flyte will run the task and store the outputs as new cached values. diff --git a/rsts/user/getting_started/create_first.rst b/rsts/user/getting_started/create_first.rst new file mode 100644 index 0000000000..9c838b82a7 --- /dev/null +++ b/rsts/user/getting_started/create_first.rst @@ -0,0 +1,122 @@ +.. _getting-started-create-first: + +######################################## +Writing Your First Workflow +######################################## + +The easiest way to author a Flyte Workflow is using the provided python SDK called "FlyteKit". + +You can save some effort by cloning the ``flytesnacks`` repo, and re-initializing it as a new git repository :: + + git clone git@github.com:lyft/flytesnacks.git myflyteproject + cd myworkflow + rm -rf .git + git init + +now open the "Makefile" and change the first line to ``IMAGE_NAME=myflyteproject`` + +Let's also remove the existing python task so we can write one from scratch. :: + + rm workflows/edges.py + +Creating a Project +****************** + +In Flyte, workflows are organized into namespaces called "Projects". When you register a workflow, it must be registered under a project. + +Lets create a new project called ``myflyteproject``. Use the project creation endpoint to create the new project :: + + curl -X POST localhost:30082/api/v1/projects -d '{"project": {"id": "myflyteproject", "name": "myflyteproject"} }' + + +Writing a Task +***************** + +The most basic Flyte primitive is a "task". Flyte Tasks are units of work that can be composed in a workflow. The simplest way to write a Flyte task is using the FlyteSDK. + +Start by creating a new file :: + + touch workflows/first.py + +This directory has been marked in the `configuration file `_ as the location to look for workflows and tasks. Begin by importing some of the libraries that we'll need for this example. + +.. code-block:: python + + from __future__ import absolute_import + from __future__ import division + from __future__ import print_function + + import urllib.request as _request + + import cv2 + from flytekit.common import utils + from flytekit.sdk.tasks import python_task, outputs, inputs + from flytekit.sdk.types import Types + from flytekit.sdk.workflow import workflow_class, Output, Input + +From there, we can begin to write our first task. It should look something like this. + +.. code-block:: python + + @inputs(image_location=Types.String) + @outputs(parsed_image=Types.Blob) + @python_task + def edge_detection_canny(wf_params, image_location, parsed_image): + with utils.AutoDeletingTempDir('test') as tmpdir: + plane_fname = '{}/plane.jpg'.format(tmpdir.name) + with _request.urlopen(image_location) as d, open(plane_fname, 'wb') as opfile: + data = d.read() + opfile.write(data) + + img = cv2.imread(plane_fname, 0) + edges = cv2.Canny(img, 50, 200) # hysteresis thresholds + + output_file = '{}/output.jpg'.format(tmpdir.name) + cv2.imwrite(output_file, edges) + + parsed_image.set(output_file) + + +Some of the new concepts demonstrated here are: + +* ``wf_params``: The first argument to a python task is a Flyte SDK defined object that offers handlers like logging. +* Inputs and outputs are first defined in the decorator, and then passed into the argument of the function. Note that the names in the function signature need to match those in the decorator arguments. +* A ``Blob`` is a Flyte Kit type that represents binary data. It is used to offload data to a storage location like S3. Here we use it to store an image. + +Writing a Workflow +********************* +Next you need to call that task from a workflow. In the same file, add these lines. + +.. code-block:: python + + @workflow_class + class EdgeDetectorWf(object): + image_input = Input(Types.String, required=True, help="Image to run for") + run_edge_detection = edge_detection_canny(image_location=image_input) + edges = Output(run_edge_detection.outputs.parsed_image, sdk_type=Types.Blob) + +This code block creates a workflow, with one task. The workflow itself has an input (the link to an image) that gets passed into the task, and an output, which is the processed image. + + +Interacting with Flyte +************************ + +Flyte fulfills tasks using docker images. You'll need to build a docker image from this code before it can run in Flyte. The repo has a make target to build the docker image for you :: + + make docker_build + +If you have the flyte sandbox installed on your local machine, the image will be accessible to to your Flyte system. If you're running a remote Flyte instance, you'll need to upload this image to a remote registry such as Dockerhub, Amazon ECR, or Google Container Registry, so that it can be used by the Flyte system. + +To upload to a remote registry, use :: + + DOCKER_REGISRY_USERNAME={username} DOCKER_REGISTRY_PASSWORD={pass} REGISTRY=docker.io make docker_build + +Replace the values above with your registry username, password, and registry endpoint. + +You may need to change the ``IMAGE_NAME`` in the Makefile to reflect your namespace in the docker registry. (ex ``{{my docker username}}/myflyteproject``) + +With the image built, we just need to register the tasks and workflows. The process is the same as what we had done previously. :: + + docker run --network host -e FLYTE_PLATFORM_URL='127.0.0.1:30081' {{ your docker image }} pyflyte -p myflyteproject -d development -c sandbox.config register workflows + +After this, you should be able to visit the Flyte UI, and run the workflow as you did with ``flytesnacks`` previously. diff --git a/rsts/user/getting_started/examples.rst b/rsts/user/getting_started/examples.rst new file mode 100644 index 0000000000..7f24584314 --- /dev/null +++ b/rsts/user/getting_started/examples.rst @@ -0,0 +1,50 @@ +###################################### +Quick Start Examples +###################################### + +.. _getting-started-examples: + +Before you can run any workflows, you'll first need to register a workflow in Flyte. We've written an example Flyte workflow, and placed it into a docker image called ``flytesnacks``. + +The example workflow takes in an image url as input, does edge detection, and produces an image showing the edges in the photo. + +Workflow Setup in Flyte +************************** + +Registration +========================== +.. _getting-started-examples-registration: + +If you're using the ``sandbox`` flyte installation, you can use the following command to register our example workflow with Flyte :: + + docker run --network host -e FLYTE_PLATFORM_URL='127.0.0.1:30081' lyft/flytesnacks:v0.1.0 pyflyte -p flytesnacks -d development -c sandbox.config register workflows + +This command will register the workflow with your Flyte app under the ``development`` domain of the project ``flytesnacks``. + +NOTE: if your Flyte endpoint is something other than ``localhost:30081``, change the ``FLYTE_PLATFORM_URL`` value accordingly. + +Running Workflows in Flyte +**************************** + +Creating an execution +========================== + +Now that your workflow is registered, you can visit the Flyte console to run the workflow. + +From the flyte console homepage http://localhost:30081/console, click the "development" link under ``flytesnacks``. This will show you the list of workflows registered under the ``development`` domain of the ``flytesnacks`` project. + +Click on the ``workflows.edges.EdgeDetectorWf`` workflow. This will take you to the Workflow Details page. + +Click the "Launch Workflow" button. This will open the Launch Workflow Form. + +Leave the ``Workflow Version`` and ``Launch Plan`` default values, but insert any publicly accessible image url in the ``image_input`` section. For example, you can use https://images.ctfassets.net/q8mvene1wzq4/1OQ8OBLzXGv8pVjFTLf0QF/9a7b8cdb982161daebd5618fc7cb5041/Car_blue_L.png. + +Click "Launch" at the bottom of the form (you may need to scroll down). + +In a few moments, you'll see the execution getting fulfilled by the Flyte system. Observe as Flyte runs through the workflow steps. + +When the workflow is complete, click the "run-edge-detection" link under "Node", this will show you some details about the execution. Click the "Outputs" tab. You should see that the workflow produced an output telling you where it stored the produced image. + +To find this image, visit the minio UI at http://localhost:30081/minio (the sandbox username is ``minio`` and the sandbox password is ``miniostorage``). + +Follow the path given in your workflow output. If you download the file, you'll see that the workflow produced the edge-detected version of the image url input. diff --git a/rsts/user/getting_started/index.rst b/rsts/user/getting_started/index.rst new file mode 100644 index 0000000000..5258d92090 --- /dev/null +++ b/rsts/user/getting_started/index.rst @@ -0,0 +1,16 @@ +.. _getting-started-2: + +Getting Started +=============== + +.. warning:: + In order to try Flyte, you must first install it. + If you do not already have access to a Flyte installation, you can install the Flyte sandbox to try Flyte. + Refer to the :ref:`Installation Guide ` to set up the Flyte sandbox. + +.. toctree:: + :maxdepth: 1 + :caption: Getting Started + + examples + create_first diff --git a/rsts/user/index.rst b/rsts/user/index.rst new file mode 100644 index 0000000000..702cf829f9 --- /dev/null +++ b/rsts/user/index.rst @@ -0,0 +1,17 @@ +.. _user: + +User docs +========= + +This doc guides you using Flyte locally or a version of Flyte hosted by your organization. + + +.. toctree:: + :maxdepth: 2 + :caption: User Documentation + :name: userdoc + + getting_started/index + concepts/index + features/index + tasktypes/index diff --git a/rsts/user/sdk/index.rst b/rsts/user/sdk/index.rst new file mode 100644 index 0000000000..b17de7ea7b --- /dev/null +++ b/rsts/user/sdk/index.rst @@ -0,0 +1,8 @@ +.. _user-sdk-python: + +Python SDK +========== + +.. toctree:: + :maxdepth: 2 + :caption: Python SDK Documentation diff --git a/rsts/user/tasktypes/container.rst b/rsts/user/tasktypes/container.rst new file mode 100644 index 0000000000..fef798caae --- /dev/null +++ b/rsts/user/tasktypes/container.rst @@ -0,0 +1,35 @@ +.. _container-task-type: + +Container Task +============== +This is one of the low-level task types. It belongs to task category "SingleStepTask", and is defined using a container image and task interface (TypedInterface). TypedInterface consists of an input map and an output map. Python Task is one implementation of this type in Flyte Python SDK. The task name and inputs are passed to the container when it is started. + +ContainerTask definition also includes RuntimeMetadata which points to the specific implementation of the "Container Task"(FlyteSDK or other). + +Python Task +----------- +Python tasks are implemented on top of container tasks. It is a python function that needs to specify all its inputs using flyte input annotation @inputs and outputs using flyte output annotation @outputs. The function is not supposed to return a value but set all the outputs before exiting. The user container built during the build step is used for registration of this container task. + + +####### +Usage +####### + +The following is an example of a simple Python task. + +.. code-block:: python + :caption: Python task example with inputs and outputs + + @inputs(value_to_print=Types.Integer) + @outputs(out=Types.Integer) + @python_task(cache_version='1') + def add_one_and_print(workflow_parameters, value_to_print, out): + added = value_to_print + 1 + workflow_parameters.logging.info("My printed value: {}".format(added)) + out.set(added) + +This does a couple things. + +* SDK uses the current container image to register this task +* SDK registers the container with special container entrypoint that knows how to run this task. + diff --git a/rsts/user/tasktypes/dynamic.rst b/rsts/user/tasktypes/dynamic.rst new file mode 100644 index 0000000000..c748549075 --- /dev/null +++ b/rsts/user/tasktypes/dynamic.rst @@ -0,0 +1,49 @@ +.. _dynamic-task-type: + +Dynamic Tasks +============= + +Flyte offers many task types to support a varity of use-cases that are all based on statically defining what +those tasks do beforehand. Dynamic Task type is the only exception to this rule where the system knows very little +about what the nature of the workload that will be executed until execution time. + +A Dynamic task is executed as follows: + - A generator step; This step runs like a :ref:`Container Task `. The expected outcome of this step is a :ref:`dynamic job spec`. + - Execute the dynamic job spec; The spec can contain any of the flyte supported workflow nodes. One or more of these nodes might contain other + dynamic tasks. In which case, it'll get recursively executed. + - Assemble final outputs of the task. + +Some of the potential use cases: + - Launching arbitrary workflows/tasks: + You might build a workflow where one of its tasks chooses and executes a launchplan based on values passed as inputs to the task. + - Dynamically creating a workflow + If the desired set of nodes/tasks that need to be executed is controlled by inputs, you can use dynamic tasks to build + a workflow spec at execution time and yield that for it to be executed. + +.. code-block:: python + :caption: Dynamic task example + + @inputs(tasks_count=Types.Integer) + @outputs(out=[Types.Integer]) + @dynamic_task(cache_version='1') + def generate_others(workflow_parameters, tasks_count, out): + res = [] + # Launch an arbitrary number of tasks based on the input tasks_count + for i in range(0, tasks_count): + task = other_task(index=i) + yield task + # Define how to assemble the final result + res.append(task.outputs.other_task_out) + + # Define how to set the final result of the task + out.set(res) + +A few notes about how this task will run: + - This code will be executed only once. Based on the way outputs are declared, :ref:`Bindings ` will + be created to instruct the system on how to assemble the final outputs after all yielded tasks are executed. + - Manipulating Outputs of the yielded tasks is not supported. Think of this step as a `Map Concept`_. If a simple reduce is + required, it'll have to happen as a separate task that consumes the assembled outputs here. + - There is a restriction on the size of individual tasks outputs as well as on the final output of this task. If large outputs + are desired, consider using Blob types. + +.. _Map Concept: https://en.wikipedia.org/wiki/MapReduce \ No newline at end of file diff --git a/rsts/user/tasktypes/hive.rst b/rsts/user/tasktypes/hive.rst new file mode 100644 index 0000000000..04dec02a67 --- /dev/null +++ b/rsts/user/tasktypes/hive.rst @@ -0,0 +1,80 @@ +.. _hive-task-type: + +============= +Hive Tasks +============= + +Hive tasks are an example of dynamic tasks. That is, they are a two-step task where the workflow container is first run to produce the queries, which are later executed using a Flyte plugin. This means that the text of the queries as well as the number of queries can be dynamic. + +See the Hive tasks discussion in the generated API documentation for more information. + + +####### +Usage +####### + +*********************** +Basic Query Execution +*********************** + +The following is an example of a simple Hive query. + +.. code-block:: python + :caption: Simple Hive query example + + @qubole_hive_task + def generate_simple_queries(wf_params): + q1 = "SELECT 1" + q2 = "SELECT 'two'" + return [q1, q2] + +This is a pretty simple query. Your queries will be run on Qubole, but nothing will happen with the output. + + +******************************* +Query with Schema Integration +******************************* + +A more common and powerful pattern is to integrate querying along with the Flyte ``Schema`` type. + +.. code-block:: python + :caption: Hive query example with Schema integration + + @outputs(hive_results=[Types.Schema()]) + @qubole_hive_task(tags=['mytag'], cluster='flyte') + def generate_queries(wf_params, hive_results): + q1 = "SELECT 1" + q2 = "SELECT 'two'" + schema_1, formatted_query_1 = Schema.create_from_hive_query(select_query=q1) + schema_2, formatted_query_2 = Schema.create_from_hive_query(select_query=q2) + + hive_results.set([schema_1, schema_2]) + return [formatted_query_1, formatted_query_2] + + +This does a couple things. + +* Your queries will be amended by the SDK before they are executed. That is, instead of ``SELECT 1``, the actual query that will be run will be something like :: + + CREATE TEMPORARY TABLE 1757c8c0d7a149b79f2c202c2c78b378_tmp AS SELECT 1; + CREATE EXTERNAL TABLE 1757c8c0d7a149b79f2c202c2c78b378 LIKE 1757c8c0d7a149b79f2c202c2c78b378_tmp STORED AS PARQUET; + ALTER TABLE 1757c8c0d7a149b79f2c202c2c78b378 SET LOCATION + \'s3://my-s3-bucket/ec/b78e1502cef04d5db8bef64a2226f707/\'; + INSERT OVERWRITE TABLE 1757c8c0d7a149b79f2c202c2c78b378 + SELECT * + FROM 1757c8c0d7a149b79f2c202c2c78b378_tmp; + DROP TABLE 1757c8c0d7a149b79f2c202c2c78b378; + + When a user's query runs, it's first selected into a temporary table, and then copied from the temporary table into the permanent external table. The external table is then dropped, which doesn't actually delete the just-queried data, but rather alleviates pressure on the Hive metastore. + +* The task's output will have been bound a priori to the location that the Qubole Hive query will end up writing to, so that the rest of Flyte (downstream tasks for instance) will know about them. + +########################### +Miscellaneous +########################### + +**************************** +Hive Execution Environment +**************************** +Qubole is currently the primary execution engine for Hive queries, though it doesn't have to be. In fact, the ``qubole_hive_task`` decorator in the SDK is a refinement on the broader ``hive_task`` decorator. + diff --git a/rsts/user/tasktypes/index.rst b/rsts/user/tasktypes/index.rst new file mode 100644 index 0000000000..d1527c1aca --- /dev/null +++ b/rsts/user/tasktypes/index.rst @@ -0,0 +1,15 @@ +.. _tasktypes: + +Flyte Task Types +================ + +.. toctree:: + :maxdepth: 2 + :name: tasktypestoc + :caption: TaskTypes + + container + hive + spark + dynamic + sidecar diff --git a/rsts/user/tasktypes/sidecar.rst b/rsts/user/tasktypes/sidecar.rst new file mode 100644 index 0000000000..9dc439e3b3 --- /dev/null +++ b/rsts/user/tasktypes/sidecar.rst @@ -0,0 +1,138 @@ +.. _sidecar-task-type: + +Sidecar Tasks +============= + +Sidecar tasks can be used to bring up multiple containers within a single task. Sidecar tasks are defined using a Kubernetes `pod spec `_ but differ slightly in that the plugin task executor solely monitors the status of a user-specified primary container over the task lifecycle. + +Installation +************ +If you haven't already, install the sidecar extra from flytekit like so: + +.. code-block:: text + + pip install flytekit[sidecar] + + +And assert that you have a dependency in your project on + +.. code-block:: text + + k8s-proto>=0.0.2 + + +Usage +##### + +Sidecar accept all arguments that ordinary :ref:`container tasks ` accept. However, sidecar tasks require two additional arguments, ``pod_spec`` and ``primary_container_name`` + +Pod Spec +******** + +Using the `generated python protobuf code in flyteproto `_, a task can define a completely kubernetes-native pod spec that will be deployed as part of the sidecar task execution. + + +Primary container +***************** + +This is a required name you use to distinguish your primary container. The code in the body of the task definition will be injected in the primary container. The pod spec you pass to the task definition does not necessarily need to include a container definition with the primary container, but if you'd like to modify the primary container by setting a shared volume mount for example, you can do so in the pod spec. + +For primary containers defined in the pod spec, a few caveats apply. The following `container `_ +fields will be overwritten at registration time and therefore are meaningless if set: + +* image +* command +* args +* resource requirements +* env + +Resource requirements and env will use the values set in the sidecar task definition and are therefore still customizable. + +For example: + +.. code-block:: python + + def get_pod_spec(): + my_pod_spec = generated_pb2.PodSpec() + my_container = generated_pb2.Container(name="primary") + # apply more customization to my_container as desired + + my_pod_spec.containers.extend([my_container]) + # apply more customization to my_pod_spec as desired + + return my_pod_spec + + + @inputs(in1=Types.String) + @outputs(out1=Types.String) + @sidecar_task( + cpu_request='10', + cpu_limit='20', + environment={"foo": "bar"}, + pod_spec=get_pod_spec(), + primary_container_name="primary", + ) + def simple_sidecar_task(wf_params, in1, out1): + print("Hi, {} I'll be run in a sidecar task!".format(in1)) + ... + + +*************** +Working Example +*************** + +For a trivial example of configuring multiple containers so that one writes to a shared volume mount and the second waits until it receives the message, see: + +.. code-block:: python + + from __future__ import absolute_import + from __future__ import print_function + + import time + import os + + from flytekit.sdk.tasks import sidecar_task + from k8s.io.api.core.v1 import generated_pb2 + + + # A simple pod spec in which a shared volume is mounted in both the primary and secondary containers. The secondary + # writes a file that the primary waits on before completing. + def generate_pod_spec_for_task(): + pod_spec = generated_pb2.PodSpec() + + primary_container = generated_pb2.Container(name="primary") + + secondary_container = generated_pb2.Container( + name="secondary", + image="alpine", + ) + secondary_container.command.extend(["/bin/sh"]) + secondary_container.args.extend(["-c", "echo hi sidecar world > /data/message.txt"]) + shared_volume_mount = generated_pb2.VolumeMount( + name="shared-data", + mountPath="/data", + ) + secondary_container.volumeMounts.extend([shared_volume_mount]) + primary_container.volumeMounts.extend([shared_volume_mount]) + + pod_spec.volumes.extend([generated_pb2.Volume( + name="shared-data", + volumeSource=generated_pb2.VolumeSource( + emptyDir=generated_pb2.EmptyDirVolumeSource( + medium="Memory", + ) + ) + )]) + pod_spec.containers.extend([primary_container, secondary_container]) + return pod_spec + + + @sidecar_task( + pod_spec=generate_pod_spec_for_task(), + primary_container_name="primary", + ) + def my_sidecar_task(wfparams): + # The code defined in this task will get injected into the primary container. + while not os.path.isfile('/data/message.txt'): + time.sleep(5) + diff --git a/rsts/user/tasktypes/spark.rst b/rsts/user/tasktypes/spark.rst new file mode 100644 index 0000000000..152b369339 --- /dev/null +++ b/rsts/user/tasktypes/spark.rst @@ -0,0 +1,51 @@ +.. _spark-task-type: + +Spark Task +========== + +Spark Task Type allows users to write a Spark Job which are then executed on the Kubernetes cluster via the `SparkOperator`_. + +####### + Setup +####### + +Spark on Kubernetes is containerized i.e. your own repo image is run as Spark driver and executors hence we need to have Spark and it's dependencies installed in the user's container. + +One way to achieve this is to build relevant images for your use-case using `Dockerfile`_ which already exist as part of the Spark distribution. `Flytekit`_ also provides *flytekit_install_spark.sh* and *flytekit_spark_entrypoint.sh* which can be used to install spark dependencies/set the correct entrypoint for Spark Tasks in an user image. + +Please refer to `Spark Documentation`_ for details on Spark on K8s. + +####### +Usage +####### + +The following is an example of a simple Spark job which calculates the value of Pi. + +.. code-block:: python + :caption: Simple Spark Job example + + @spark_task( + spark_conf={ + 'spark.driver.memory': "1000M", + 'spark.executor.memory': "1000M", + 'spark.executor.cores': '1', + 'spark.executor.instances': '2', + }, + ) + def hello_spark(workflow_parameters, spark_context, partitions, out): + print("Starting Spark with Partitions: {}".format(partitions)) + n = 100000 * partitions + count = spark_context.parallelize(range(1, n + 1), partitions).map(f).reduce(add) + pi_val = 4.0 * count / n + print("Pi val is :{}".format(pi_val)) + out.set(pi_val) + +This is a pretty simple job. It calculates the value of Pi and sets it as the output for the task. The task takes in `spark_conf` and `hadoop_conf` where the user can provide any custom spark and hadoop config required for the Spark Task. + +`Flyte Workflow Demo - Spark Workflow`_ is a Spark Workflow example which uses the Spark Task defined above. + +.. _SparkOperator: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator +.. _Dockerfile: https://github.com/apache/spark/blob/master/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile#L22 +.. _Spark Documentation: https://spark.apache.org/docs/latest/running-on-kubernetes.html +.. _Flytekit: https://github.com/lyft/flytekit/tree/master/scripts +.. _Flyte Workflow Demo - Spark Workflow: https://github.com/lyft/flytekit/blob/master/tests/flytekit/common/workflows/spark.py diff --git a/script/deploy.sh b/script/deploy.sh new file mode 100755 index 0000000000..dcd1717101 --- /dev/null +++ b/script/deploy.sh @@ -0,0 +1,6 @@ +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +DEPLOY_FILE="${DIR}/../deployment/sandbox/flyte_generated.yaml" +kubectl apply -f "$DEPLOY_FILE" diff --git a/script/flyte_venv.sh b/script/flyte_venv.sh new file mode 100755 index 0000000000..fae6e3618a --- /dev/null +++ b/script/flyte_venv.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Our SDK entrypoint can be configured to call this script +# This script maps that command to the conventional location of the virtual environment in Flyte containers + +set -e + +. ${VENV}/bin/activate + +exec $* diff --git a/script/generate_docs.sh b/script/generate_docs.sh new file mode 100755 index 0000000000..88261d3653 --- /dev/null +++ b/script/generate_docs.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Note that this file is meant to be run on OSX by a user with the necessary GitHub privileges + +set -ex + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +BASEDIR="${DIR}/.." + +# Clean out all old docs, generate everything from scratch every time +rm -rf ${BASEDIR}/docs + +# Set up a temp directory +rm -rf ${BASEDIR}/rsts_tmp || true +mkdir ${BASEDIR}/rsts_tmp || true +RSTS_DIR=`mktemp -d "${BASEDIR}/rsts_tmp/XXXXXXXXX"` + +# Copy all rst files to the same place +cp -R rsts/* ${RSTS_DIR} +cp -R _rsts/* ${RSTS_DIR} + +# The toctree in this index file requires that the idl/sdk rsts are in the same folder +cp docs_infra/index.rst ${RSTS_DIR} + +# Generate documentation by running script inside the generation container +docker run -t -v ${BASEDIR}:/base -v ${BASEDIR}/docs:/docs -v ${RSTS_DIR}:/rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_html_generation.sh + +# Cleanup +rm -rf ${RSTS_DIR} || true diff --git a/script/kustomize.sh b/script/kustomize.sh new file mode 100755 index 0000000000..1b89c6199b --- /dev/null +++ b/script/kustomize.sh @@ -0,0 +1,10 @@ +set -ex + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +KUSTOMIZE_IMAGE="lyft/kustomizer:v3.1.0" + +# flyte test +docker run -v "${DIR}/../kustomize":/kustomize "$KUSTOMIZE_IMAGE" kustomize build overlays/test/flyte > "${DIR}/../deployment/test/flyte_generated.yaml" + +# flyte local +docker run -v "${DIR}/../kustomize":/kustomize "$KUSTOMIZE_IMAGE" kustomize build overlays/sandbox/flyte > "${DIR}/../deployment/sandbox/flyte_generated.yaml" diff --git a/script/update_ref_docs.sh b/script/update_ref_docs.sh new file mode 100755 index 0000000000..f31cf6a131 --- /dev/null +++ b/script/update_ref_docs.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Note that this file is meant to be run on OSX by a user with the necessary GitHub privileges + +set -ex + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +BASEDIR="${DIR}/.." + +# Set up a temp directory +mkdir ${BASEDIR}/_repos || true +REPOS_DIR=`mktemp -d "${BASEDIR}/_repos/XXXXXXXXX"` + +function clone_repos() +{ + git clone https://github.com/lyft/flytekit.git ${REPOS_DIR}/flytekit + git clone https://github.com/lyft/flyteidl.git ${REPOS_DIR}/flyteidl +} + +# Clone all repos +$(clone_repos) + +# Generate documentation by running script inside the generation container +docker run -t -v ${BASEDIR}:/base -v ${REPOS_DIR}:/repos -v ${BASEDIR}/_rsts:/_rsts lyft/docbuilder:v2.2.0 /base/docs_infra/in_container_rst_generation.sh + +# Cleanup +rm -rf ${REPOS_DIR}/* || true