From 8f1ea41e5142b690a7de8c02e6e9fabcac223034 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Tue, 30 Mar 2021 11:39:49 -0400 Subject: [PATCH 01/92] update docs theme (#876) * update docs theme Signed-off-by: cosmicBboy * remove sphinx material and readthedocs search Signed-off-by: cosmicBboy * update flytekit link Signed-off-by: cosmicBboy * update Signed-off-by: cosmicBboy * add readthedocs sphinx search Signed-off-by: cosmicBboy * update community link Signed-off-by: cosmicBboy Signed-off-by: Haytham Abuelfutuh --- doc-requirements.in | 3 +- doc-requirements.txt | 42 ++++++-------- rsts/_static/custom.css | 11 ++++ rsts/conf.py | 66 +++++++++------------- rsts/getting_started.rst | 74 +++++++++++++++++++++++++ rsts/getting_started/first_example.rst | 60 -------------------- rsts/getting_started/first_run.rst | 26 --------- rsts/getting_started/learn.rst | 8 --- rsts/getting_started/run_on_flyte.rst | 77 -------------------------- rsts/index.rst | 67 ++++++++++++++-------- rsts/reference/index.rst | 15 +++-- 11 files changed, 182 insertions(+), 267 deletions(-) create mode 100644 rsts/_static/custom.css create mode 100644 rsts/getting_started.rst delete mode 100644 rsts/getting_started/first_example.rst delete mode 100644 rsts/getting_started/first_run.rst delete mode 100644 rsts/getting_started/learn.rst delete mode 100644 rsts/getting_started/run_on_flyte.rst diff --git a/doc-requirements.in b/doc-requirements.in index d84752e50a..79fbabb86a 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -1,6 +1,7 @@ +furo +readthedocs-sphinx-search sphinx sphinx-prompt -sphinx-material sphinx-code-include sphinx-autoapi sphinx-copybutton diff --git a/doc-requirements.txt b/doc-requirements.txt index b7e13e0c63..a801a17fd3 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -6,22 +6,22 @@ # alabaster==0.7.12 # via sphinx -astroid==2.4.2 +astroid==2.5.2 # via sphinx-autoapi babel==2.9.0 # via sphinx beautifulsoup4==4.9.3 # via + # furo # sphinx-code-include - # sphinx-material certifi==2020.12.5 # via requests chardet==4.0.0 # via requests -css-html-js-minify==2.5.5 - # via sphinx-material docutils==0.16 # via sphinx +furo==2021.3.20b30 + # via -r doc-requirements.in idna==2.10 # via requests imagesize==1.2.0 @@ -30,36 +30,34 @@ jinja2==2.11.3 # via # sphinx # sphinx-autoapi -lazy-object-proxy==1.4.3 +lazy-object-proxy==1.6.0 # via astroid -lxml==4.6.2 - # via sphinx-material markupsafe==1.1.1 # via jinja2 packaging==20.9 # via sphinx -pygments==2.7.4 +pygments==2.8.1 # via # sphinx # sphinx-prompt # sphinx-tabs pyparsing==2.4.7 # via packaging -python-slugify[unidecode]==4.0.1 - # via sphinx-material pytz==2021.1 # via babel pyyaml==5.4.1 # via sphinx-autoapi +readthedocs-sphinx-search==0.1.0 + # via -r doc-requirements.in requests==2.25.1 # via sphinx six==1.15.0 # via - # astroid # sphinx-code-include + # sphinxext-remoteliteralinclude snowballstemmer==2.1.0 # via sphinx -soupsieve==2.2 +soupsieve==2.2.1 # via beautifulsoup4 sphinx-autoapi==1.7.0 # via -r doc-requirements.in @@ -69,20 +67,18 @@ sphinx-copybutton==0.3.1 # via -r doc-requirements.in sphinx-issues==1.2.0 # via -r doc-requirements.in -sphinx-material==0.0.32 - # via -r doc-requirements.in -sphinx-prompt==1.3.0 +sphinx-prompt==1.4.0 # via -r doc-requirements.in -sphinx-tabs==2.0.1 +sphinx-tabs==2.1.0 # via -r doc-requirements.in -sphinx==3.4.3 +sphinx==3.5.3 # via # -r doc-requirements.in + # furo # sphinx-autoapi # sphinx-code-include # sphinx-copybutton # sphinx-issues - # sphinx-material # sphinx-prompt # sphinx-tabs # sphinxext-remoteliteralinclude @@ -98,15 +94,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx -sphinxext-remoteliteralinclude==0.1.0 +sphinxext-remoteliteralinclude==0.2.0 # via -r doc-requirements.in -text-unidecode==1.3 - # via python-slugify unidecode==1.2.0 - # via - # python-slugify - # sphinx-autoapi -urllib3==1.26.3 + # via sphinx-autoapi +urllib3==1.26.4 # via requests wrapt==1.12.1 # via astroid diff --git a/rsts/_static/custom.css b/rsts/_static/custom.css new file mode 100644 index 0000000000..13c39dba3f --- /dev/null +++ b/rsts/_static/custom.css @@ -0,0 +1,11 @@ +h1, h2, h3, h4, h5, h6 { + font-weight: bold; +} + +.sidebar-logo { + max-width: 30%; +} + +.sidebar-tree .reference.external:after { + content: none; +} diff --git a/rsts/conf.py b/rsts/conf.py index 871f9190d5..9bfce8e5e9 100644 --- a/rsts/conf.py +++ b/rsts/conf.py @@ -16,6 +16,10 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) +import sphinx.application +import sphinx.errors +sphinx.application.ExtensionError = sphinx.errors.ExtensionError + # -- Project information ----------------------------------------------------- project = u'Flyte' @@ -55,6 +59,7 @@ "sphinx_tabs.tabs", "sphinxext.remoteliteralinclude", "sphinx_issues", + "sphinx_search.extension", ] extlinks = { @@ -97,45 +102,28 @@ # html_favicon = "images/flyte_circle_gradient_1_4x4.png" html_logo = "images/flyte_circle_gradient_1_4x4.png" -html_static_path = [] -html_theme = "sphinx_material" +html_theme = "furo" +html_title = "Flyte Docs" + +html_static_path = ["_static"] +templates_path = ["_templates"] + +pygments_style = "tango" +pygments_dark_style = "paraiso-dark" + +html_css_files = [ + "custom.css", +] + html_theme_options = { - # Set the name of the project to appear in the navigation. - "nav_title": "Flyte", - # Set you GA account ID to enable tracking - "google_analytics_account": "G-YQL24L5CKY", - # Specify a base_url used to generate sitemap.xml. If not - # specified, then no sitemap will be built. - "base_url": "https://github.com/flyteorg/flyte", - # Set the color and the accent color - "color_primary": "deep-purple", - "color_accent": "blue", - # Set the repo location to get a badge with stats - "repo_url": "https://github.com/flyteorg/flyte/", - "repo_name": "flyte", - # Visible levels of the global TOC; -1 means unlimited - "globaltoc_depth": 1, - # If False, expand all TOC entries - "globaltoc_collapse": False, - # If True, show hidden TOC entries - "globaltoc_includehidden": True, - # don't include home link in breadcrumb bar, since it's included - # in the nav_links key below. - "master_doc": False, - # custom nav in breadcrumb bar - "nav_links": [ - {"href": "index", "internal": True, "title": "Flyte"}, - { - "href": "https://flytecookbook.readthedocs.io", - "internal": False, - "title": "Tutorials", - }, - { - "href": "https://flytekit.readthedocs.io", - "internal": False, - "title": "Flytekit Python Reference" - }, - ], + "light_css_variables": { + "color-brand-primary": "#4300c9", + "color-brand-content": "#4300c9", + }, + "dark_css_variables": { + "color-brand-primary": "#9D68E4", + "color-brand-content": "#9D68E4", + }, } # Theme options are theme-specific and customize the look and feel of a theme @@ -157,7 +145,7 @@ # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', # 'searchbox.html']``. # -html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} +# html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} # -- Options for HTMLHelp output --------------------------------------------- diff --git a/rsts/getting_started.rst b/rsts/getting_started.rst new file mode 100644 index 0000000000..09b8e7c545 --- /dev/null +++ b/rsts/getting_started.rst @@ -0,0 +1,74 @@ +.. _gettingstarted: + +Getting started +--------------- + +.. rubric:: Estimated time: 3 minutes + +Prerequisites +############# + +Make sure you have `docker installed `__ and `git `__ installed, then install flytekit: + +.. prompt:: bash + + pip install flytekit + +Clone the `flytekit-python-template `__ repo to create our own git repository called ``flyteexamples``: + +.. prompt:: bash + + git clone git@github.com:flyteorg/flytekit-python-template.git flyteexamples + cd flyteexamples + + +Write Your First Flyte Workflow +############################### + + +Let's take a look at the example workflow in `myapp/workflows/example.py `__: + +.. rli:: https://raw.githubusercontent.com/flyteorg/flytekit-python-template/main/myapp/workflows/example.py + :language: python + +As you can see, a Flyte :std:doc:`task ` is the most basic unit of work in Flyte, +and you can compose multiple tasks into a :std:doc:`workflow `. Try running and +modifying the ``example.py`` script locally. + +Start a Local Flyte Backend +########################### + +Once you're happy with the ``example.py`` script, run the following command in your terminal: + +.. prompt:: bash + + docker run --rm --privileged -p 30081:30081 -p 30082:30082 -p 30084:30084 ghcr.io/flyteorg/flyte-sandbox + +When you see the message ``Flyte is ready!``, your local sandbox should be ready on http://localhost:30081/console. + +Register Your Workflows +########################### + +Now we're ready to ship your code to the Flyte backend by running the following command: + +.. prompt:: bash + + FLYTE_AWS_ENDPOINT=http://localhost:30084/ FLYTE_AWS_ACCESS_KEY_ID=minio FLYTE_AWS_SECRET_ACCESS_KEY=miniostorage make fast_register + +Run Your Workflows +################## + +To run a workflow, go to http://localhost:30081/console/projects/flyteexamples/workflows and then follow these steps: + +1. Select the ``hello_world`` workflow +2. Click the **Launch Workflow** button in the upper right corner +3. Update the ``name`` input argument +4. Proceed to **Launch** to trigger an execution + +.. rubric:: 🎉 Congratulations, you just ran your first Flyte workflow 🎉 + + +Next Steps: Tutorials +##################### + +To experience the full capabilities of Flyte, try out the `Flytekit Tutorials `__ 🛫 diff --git a/rsts/getting_started/first_example.rst b/rsts/getting_started/first_example.rst deleted file mode 100644 index fc6c0e09b7..0000000000 --- a/rsts/getting_started/first_example.rst +++ /dev/null @@ -1,60 +0,0 @@ -.. _getting-started-first-example: - -############################### -Write Your First Flyte Workflow -############################### - -By the end of this guide you will become familiar with how easy it is to author a Flyte workflow and run it locally. - -.. rubric:: Estimated time: <3 minutes - - -Prerequisites -************* - -#. Ensure that you have `git `__ installed. - -#. Let us make a virutal environment **(recommended)** - and then install flytekit using - ``--pre`` is used because we are currently using the beta version of flytekit 0.16.0, this introduces a completely new SDK for authoring workflows - :: - - pip install --pre flytekit - - -#. Now we will use the ``flytekit-python-template`` repo to create our own git repository called ``flyteexamples`` :: - - git clone git@github.com:flyteorg/flytekit-python-template.git flyteexamples - cd flyteexamples - rm -rf .git - git init - - -Flyte Tasks and Workflows -************************* - -Take a look at the example workflow found in `myapp/workflows/example.py `__ - -.. rli:: https://raw.githubusercontent.com/flyteorg/flytekit-python-template/main/myapp/workflows/example.py - :language: python - -The most basic Flyte primitive is a :std:doc:`task ` - -Flyte tasks are units of work that can be composed in a :std:doc:`workflow ` - -You can call this task: - -.. code-block:: python - - greet(name="world") - -and iterate it locally before adding it to part of a larger overall workflow. - -Similarly, you can call this workflow: - -.. code-block:: python - - hello_world(name=...) - -and iterate it locally before registering it with Flyte. - -.. tip:: Every invocation of a Flyte workflow requires specifying keyword arguments as in the example - ``hello_world(name="name")``. Calling the workflow without the keyword ``name`` will raise an ``AssertionError``. diff --git a/rsts/getting_started/first_run.rst b/rsts/getting_started/first_run.rst deleted file mode 100644 index 404a3a7fda..0000000000 --- a/rsts/getting_started/first_run.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. currentmodule:: firstrun - -.. _getting-started-firstrun: - -########################### -Start a Local Flyte Backend -########################### - -.. rubric:: Estimated time: 2 minutes - -Flyte enables scalable, reproducible and reliable orchestration of massively large workflows. To get a sense of the product, a minimalist version of the Flyte system is packaged into a Docker image. - -With `docker installed `__, run this command: :: - - docker run --rm --privileged -p 30081:30081 -p 30082:30082 -p 30084:30084 ghcr.io/flyteorg/flyte-sandbox - -This creates a local Flyte sandbox. Once the sandbox is ready, you should see the following message: - -``Flyte is ready! Flyte UI is available at http://localhost:30081/console``. - -Go ahead and visit http://localhost:30081/console to check it out. - -Below is a quick visual tour for launching your first Workflow: - -.. image:: https://github.com/flyteorg/flyte/raw/static-resources/img/first-run-console-2.gif - :alt: A quick visual tour for launching your first Workflow. diff --git a/rsts/getting_started/learn.rst b/rsts/getting_started/learn.rst deleted file mode 100644 index dad96a1142..0000000000 --- a/rsts/getting_started/learn.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _getting-started-learn: - -############################## -Next Steps: Flytekit Tutorials -############################## - -Flytekit provides programmatic access to Flyte's functionality, with SDK support in Python and Java. -The best way to see the capabilities of Flyte is by trying out the `Flytekit Tutorials `__. diff --git a/rsts/getting_started/run_on_flyte.rst b/rsts/getting_started/run_on_flyte.rst deleted file mode 100644 index 423e111669..0000000000 --- a/rsts/getting_started/run_on_flyte.rst +++ /dev/null @@ -1,77 +0,0 @@ -.. _getting-started-run-on-flyte: - -##################################### -Run Your Workflow on a Flyte Backend -##################################### - -************************ -Installing Flyte Locally -************************ - -This guide will walk you through: - -* A quick installation of Flyte on your device - -* How to register and execute your workflows against this deployment. - -(The tips below have an option to install Flyte on a cloud provider as well) - -.. rubric:: Estimated time: 1 minute - -Prerequisites -============= - -1. Follow :ref:`getting-started-firstrun` - -2. Access FlyteConsole on http://localhost:30081/console - -.. tip:: To check if your flyte-sandbox container is running you can run ``docker ps`` and it should show image ``ghcr.io/flyteorg/flyte-sandbox`` running - -.. tip:: If you prefer using k3d, Minikube, docker for mac, or a hosted Kubernetes cluster like AWS-EKS, GCP-GKE, Azure Kubernetes refer to :ref:`howto-sandbox`. It is recommended that you use a simple Docker based approach when you are first getting started with Flyte. - -.. _getting-started-run-flyte-laptop: - -**************************** -Running Your Flyte Workflows -**************************** - -Register Your Workflows -======================= -Registration is the process of shipping your code to the Flyte backend. This creates an immutable, versioned record of your code with the FlyteAdmin service. - -From within root directory of ``flyteexamples`` you previously created with the :ref:`Write Your First Workflow tutorial `, -feel free to make any changes and then register: :: - - FLYTE_AWS_ENDPOINT=http://localhost:30084/ FLYTE_AWS_ACCESS_KEY_ID=minio \ - FLYTE_AWS_SECRET_ACCESS_KEY=miniostorage make fast_register - - -.. tip:: Flyte sandbox uses minio as a substitute for S3/GCS etc. It is port-forwarded in the first command to 30084. If you use S3/GCS or a different port-forward you can drop or change the ``FLYTE_AWS_ENDPOINT`` accordingly. - -.. rubric:: It's that simple! - -Run Your Workflows -================== - -Visit the page housing workflows registered to your project at: -`http://localhost:30081/console/projects/flyteexamples/workflows `__ - -Once you have accessed your workflows, - -* Select your workflow -* Click the bright purple "Launch Workflow" button in the upper right -* Update the "name" input argument -* Proceed to launch to trigger an execution - -.. note:: - - After registration, Flyte Workflows exist in the FlyteAdmin service and can be triggered using: - - Console - - Command line - - Directly invoking the REST API - - On a schedule - - -Create a New Project --------------------- -Visit :ref:`howto_new_project`. diff --git a/rsts/index.rst b/rsts/index.rst index ee33251783..4bd7d9cafd 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -1,18 +1,14 @@ -Flyte -===== -.. image:: images/flyte_lockup_gradient_on_light.png .. toctree:: :maxdepth: 1 - :caption: Getting Started - :name: gettingstartedtoc + :name: mainsections :hidden: - getting_started/first_run - getting_started/first_example - getting_started/run_on_flyte - getting_started/learn + getting_started + Tutorials + reference/index + community/index .. toctree:: :caption: How-Tos @@ -30,15 +26,14 @@ Flyte :hidden: dive_deep/index - reference/index .. toctree:: - :caption: Contributor Guide + :caption: Developers :maxdepth: 1 :name: roadmaptoc :hidden: - community/index + Join the Community community/docs community/roadmap community/compare @@ -48,17 +43,40 @@ Meet Flyte Flyte is an open-source, container-native, structured programming and distributed processing platform. It enables highly concurrent, scalable and maintainable workflows for machine learning and data processing. -Created at Lyft, Flyte provides first class support for Python, Java, and Scala, and is built directly on Kubernetes for all the benefits containerization provides: portability, scalability, and reliability. +Created at `Lyft `__, Flyte provides first class support for Python, Java, and Scala, and is built directly on Kubernetes for all the benefits containerization provides: portability, scalability, and reliability. + +The core unit of execution in Flyte is the ``task``, which you can easily write with the Flytekit Python SDK: + +.. code:: python + + @task + def greet(name: str) -> str: + return f"Welcome, {name}!" + +You can compose one or more tasks to create a ``workflow``: + +.. code:: python + + @task + def add_question(greeting: str) -> str: + return f"{greeting} How are you?" + + @workflow + def welcome(name: str) -> str: + greeting = greet(name=name) + return add_question(greeting=greeting) + + welcome("Traveler") + # Output: "Welcome, Traveler! How are you?" -Flyte provides a single unit of execution (task) as a top-level concept. Multiple tasks arranged in a data producer-consumer order create the workflow, which is pure specification created in any language. Why Flyte? ========== Flyte's main purpose is to increase the development velocity for data processing and machine learning, enabling large-scale compute execution without the operational overhead. Teams can therefore focus on the business goals and not the infrastructure. -What makes Flyte different? ---------------------------- +Core Features +------------- * Container Native * Extensible Backend & SDK’s @@ -71,14 +89,17 @@ What makes Flyte different? * Operation Visibility - cost, performance, etc. * Cross-Cloud Portable Pipelines -At Lyft, Flyte has served production model training and data processing for over four years, becoming the de-facto platform for the Pricing, Locations, ETA, Mapping teams, Growth, Autonomous and other teams - -Whether you will be writing Flyte workflows, deploying the Flyte platform to your k8 cluster, or would like to extend and contribute to the architecture and design of Flyte, we have what you need. +Who's Using Flyte? +------------------ -Welcome to the documentation hub for Flyte! +At `Lyft `__, Flyte has served production model training and data processing for over four years, becoming the de-facto platform for the Pricing, Locations, ETA, Mapping teams, Growth, Autonomous and other teams. -Beginners: see :ref:`getting-started-firstrun` +Next Steps +---------- -Intermediate Users: see :ref:`plugins` +Whether you want to write Flyte workflows, deploy the Flyte platform to your k8 cluster, or extend and contribute to the architecture and design of Flyte, we have what you need. -Advanced Users: see :ref:`divedeep` +* :ref:`Get Started ` +* :ref:`Main Concepts ` +* :ref:`Extend Flyte ` +* :ref:`Join the Community ` diff --git a/rsts/reference/index.rst b/rsts/reference/index.rst index 32281a70c4..c27aeaa579 100644 --- a/rsts/reference/index.rst +++ b/rsts/reference/index.rst @@ -1,8 +1,8 @@ .. _reference: -################################# -API, CLI and Component Reference -################################# +############# +API Reference +############# .. toctree:: @@ -10,10 +10,10 @@ API, CLI and Component Reference :caption: API Reference :name: apitoc - Flyte Core Language Specification (FlyteIDL) - Flytekit Python - Flytekit JAVA - Flytectl CLI reference + Flytekit Python + Flytekit Java + FlyteIDL + Flytectl .. toctree:: @@ -25,4 +25,3 @@ API, CLI and Component Reference FlyteAdmin FlytePlugins DataCatalog - Flytectl \ No newline at end of file From be8db595dde018c3eefa27b5bb6112f2375f7e7d Mon Sep 17 00:00:00 2001 From: Ruslan Stanevich Date: Tue, 13 Oct 2020 21:58:18 +0300 Subject: [PATCH 02/92] Helm chart for Flyte - initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/.helmignore | 23 + helm/Chart.yaml | 6 + helm/README.MD | 11 + helm/templates/_helpers.tpl | 216 ++++++ helm/templates/admin/configmap.yaml | 22 + helm/templates/admin/cronjob.yaml | 41 ++ helm/templates/admin/deployment.yaml | 133 ++++ helm/templates/admin/rbac.yaml | 52 ++ helm/templates/admin/service.yaml | 27 + helm/templates/common/configmap.yaml | 25 + helm/templates/common/ingress.yaml | 40 ++ helm/templates/common/namespace.yaml | 4 + helm/templates/common/secret.yaml | 4 + helm/templates/console/configmap.yaml | 7 + helm/templates/console/deployment.yaml | 44 ++ helm/templates/console/service.yaml | 18 + .../templates/contour/crds/ingressroutes.yaml | 136 ++++ helm/templates/contour/deployment.yaml | 83 +++ helm/templates/contour/namespace.yaml | 6 + helm/templates/contour/rbac.yaml | 78 +++ helm/templates/contour/service.yaml | 17 + helm/templates/datacatalog/configmap.yaml | 14 + helm/templates/datacatalog/deployment.yaml | 69 ++ helm/templates/datacatalog/rbac.yaml | 11 + helm/templates/datacatalog/service.yaml | 27 + helm/templates/minio/deployment.yaml | 50 ++ helm/templates/minio/service.yaml | 21 + helm/templates/postgres/deployment.yaml | 42 ++ helm/templates/postgres/service.yaml | 21 + helm/templates/propeller/configmap.yaml | 32 + .../propeller/crds/flyteworkflow.yaml | 14 + helm/templates/propeller/deployment.yaml | 55 ++ helm/templates/propeller/rbac.yaml | 88 +++ .../pytorch-operator/crds/pytorchjobs.yaml | 44 ++ .../pytorch-operator/deployment.yaml | 48 ++ .../templates/pytorch-operator/namespace.yaml | 6 + helm/templates/pytorch-operator/rbac.yaml | 108 +++ helm/templates/pytorch-operator/service.yaml | 20 + helm/templates/redis/service.yaml | 21 + helm/templates/redis/statefulset.yaml | 70 ++ .../crds/scheduledsparkapplications.yaml | 103 +++ .../sparkoperator/crds/sparkapplications.yaml | 149 ++++ helm/templates/sparkoperator/deployment.yaml | 59 ++ helm/templates/sparkoperator/job.yaml | 28 + helm/templates/sparkoperator/namespace.yaml | 6 + helm/templates/sparkoperator/rbac.yaml | 167 +++++ helm/templates/sparkoperator/service.yaml | 20 + helm/values-eks.yaml | 645 ++++++++++++++++++ helm/values-sandbox.yaml | 146 ++++ helm/values.yaml | 550 +++++++++++++++ 50 files changed, 3627 insertions(+) create mode 100644 helm/.helmignore create mode 100644 helm/Chart.yaml create mode 100644 helm/README.MD create mode 100644 helm/templates/_helpers.tpl create mode 100644 helm/templates/admin/configmap.yaml create mode 100644 helm/templates/admin/cronjob.yaml create mode 100644 helm/templates/admin/deployment.yaml create mode 100644 helm/templates/admin/rbac.yaml create mode 100644 helm/templates/admin/service.yaml create mode 100644 helm/templates/common/configmap.yaml create mode 100644 helm/templates/common/ingress.yaml create mode 100644 helm/templates/common/namespace.yaml create mode 100644 helm/templates/common/secret.yaml create mode 100644 helm/templates/console/configmap.yaml create mode 100644 helm/templates/console/deployment.yaml create mode 100644 helm/templates/console/service.yaml create mode 100644 helm/templates/contour/crds/ingressroutes.yaml create mode 100644 helm/templates/contour/deployment.yaml create mode 100644 helm/templates/contour/namespace.yaml create mode 100644 helm/templates/contour/rbac.yaml create mode 100644 helm/templates/contour/service.yaml create mode 100644 helm/templates/datacatalog/configmap.yaml create mode 100644 helm/templates/datacatalog/deployment.yaml create mode 100644 helm/templates/datacatalog/rbac.yaml create mode 100644 helm/templates/datacatalog/service.yaml create mode 100644 helm/templates/minio/deployment.yaml create mode 100644 helm/templates/minio/service.yaml create mode 100644 helm/templates/postgres/deployment.yaml create mode 100644 helm/templates/postgres/service.yaml create mode 100644 helm/templates/propeller/configmap.yaml create mode 100644 helm/templates/propeller/crds/flyteworkflow.yaml create mode 100644 helm/templates/propeller/deployment.yaml create mode 100644 helm/templates/propeller/rbac.yaml create mode 100644 helm/templates/pytorch-operator/crds/pytorchjobs.yaml create mode 100644 helm/templates/pytorch-operator/deployment.yaml create mode 100644 helm/templates/pytorch-operator/namespace.yaml create mode 100644 helm/templates/pytorch-operator/rbac.yaml create mode 100644 helm/templates/pytorch-operator/service.yaml create mode 100644 helm/templates/redis/service.yaml create mode 100644 helm/templates/redis/statefulset.yaml create mode 100644 helm/templates/sparkoperator/crds/scheduledsparkapplications.yaml create mode 100644 helm/templates/sparkoperator/crds/sparkapplications.yaml create mode 100644 helm/templates/sparkoperator/deployment.yaml create mode 100644 helm/templates/sparkoperator/job.yaml create mode 100644 helm/templates/sparkoperator/namespace.yaml create mode 100644 helm/templates/sparkoperator/rbac.yaml create mode 100644 helm/templates/sparkoperator/service.yaml create mode 100644 helm/values-eks.yaml create mode 100644 helm/values-sandbox.yaml create mode 100644 helm/values.yaml diff --git a/helm/.helmignore b/helm/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/Chart.yaml b/helm/Chart.yaml new file mode 100644 index 0000000000..aef2a99bd4 --- /dev/null +++ b/helm/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: flyte +description: A Helm chart for Flyte +type: application +version: 0.1.0 +appVersion: 0.8.0 diff --git a/helm/README.MD b/helm/README.MD new file mode 100644 index 0000000000..bd8379c8b7 --- /dev/null +++ b/helm/README.MD @@ -0,0 +1,11 @@ +### SANDBOX INSTALLATION: +- Install `helm 3` realese version +- Generate `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` +- Deploy the manifest `kubectl apply -f flyte_generated_sandbox.yaml` +- When all pods are running - run end2end tests: `kubectl apply -f ../end2end/tests/endtoend.yaml` +- Get flyte host `minikube service contour -n heptio-contour --url`. And then visit `http:///console` + +### CONFIGURATION NOTES: +- The docker images, their tags and other default parameters are configured in `values.yaml` file. +- Each Flyte installation type should have separate `values-*.yaml` file: for sandbox, EKS and etc. The configuration in `values.yaml` and the choosen config `values-*.yaml` are merged when generating the deployment manifest. +- The configuration in `values-sandbox.yaml` is ready for installation in minikube. But `values-eks.yaml` should be edited before installation: s3 bucket, RDS hosts, iam roles, secrets and etc need to be modified. diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl new file mode 100644 index 0000000000..189f8f89e8 --- /dev/null +++ b/helm/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* vim: set filetype=mustache: */}} + + + +{{- define "flyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "flyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "flyte.namespace" -}} +{{- default .Release.Namespace .Values.forceNamespace | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + + +{{- define "flyteadmin.name" -}} +flyteadmin +{{- end -}} + +{{- define "flyteadmin.selectorLabels" -}} +app.kubernetes.io/name: {{ template "flyteadmin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "flyteadmin.labels" -}} +{{ include "flyteadmin.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + + +{{- define "datacatalog.name" -}} +datacatalog +{{- end -}} + +{{- define "datacatalog.selectorLabels" -}} +app.kubernetes.io/name: {{ template "datacatalog.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "datacatalog.labels" -}} +{{ include "datacatalog.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + + +{{- define "flytepropeller.name" -}} +flytepropeller +{{- end -}} + +{{- define "flytepropeller.selectorLabels" -}} +app.kubernetes.io/name: {{ template "flytepropeller.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "flytepropeller.labels" -}} +{{ include "flytepropeller.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + + +{{- define "flyteconsole.name" -}} +flyteconsole +{{- end -}} + +{{- define "flyteconsole.selectorLabels" -}} +app.kubernetes.io/name: {{ template "flyteconsole.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "flyteconsole.labels" -}} +{{ include "flyteconsole.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + + +{{- define "redis.name" -}} +redis +{{- end -}} + +{{- define "redis.selectorLabels" -}} +app.kubernetes.io/name: {{ template "redis.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "redis.labels" -}} +{{ include "redis.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + + +{{- define "postgres.name" -}} +postgres +{{- end -}} + +{{- define "postgres.selectorLabels" -}} +app.kubernetes.io/name: {{ template "postgres.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "postgres.labels" -}} +{{ include "postgres.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + + +{{- define "minio.name" -}} +minio +{{- end -}} + +{{- define "minio.selectorLabels" -}} +app.kubernetes.io/name: {{ template "minio.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "minio.labels" -}} +{{ include "minio.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + +{{- define "contour.name" -}} +contour +{{- end -}} + +{{- define "contour.namespace" -}} +heptio-contour +{{- end -}} + +{{- define "contour.selectorLabels" -}} +app.kubernetes.io/name: {{ template "contour.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "contour.labels" -}} +{{ include "contour.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + + +{{- define "sparkoperator.name" -}} +sparkoperator +{{- end -}} + +{{- define "sparkoperator.namespace" -}} +sparkoperator +{{- end -}} + +{{- define "sparkoperator.selectorLabels" -}} +app.kubernetes.io/name: {{ template "sparkoperator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "sparkoperator.labels" -}} +{{ include "sparkoperator.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + + +{{- define "pytorch-operator.name" -}} +pytorch-operator +{{- end -}} + +{{- define "pytorch-operator.namespace" -}} +pytorch-operator +{{- end -}} + +{{- define "pytorch-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ template "pytorch-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "pytorch-operator.labels" -}} +{{ include "pytorch-operator.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + + +# +# SECRET +# + +{{- define "databaseSecret.volume" -}} +{{- with .Values.common.databaseSecret.name -}} +- name: {{ . }} + secret: + secretName: {{ . }} +{{- end }} +{{- end }} + +{{- define "databaseSecret.volumeMount" -}} +{{- with .Values.common.databaseSecret.name -}} +- mountPath: /etc/db + name: {{ . }} +{{- end }} +{{- end }} diff --git a/helm/templates/admin/configmap.yaml b/helm/templates/admin/configmap.yaml new file mode 100644 index 0000000000..75921682d2 --- /dev/null +++ b/helm/templates/admin/configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-admin-config + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} +data: + cluster_resources.yaml: | {{ toYaml .Values.configmap.cluster_resources | nindent 4 }} + + db.yaml: | {{ toYaml .Values.configmap.db | nindent 4 }} + + domain.yaml: | {{ toYaml .Values.configmap.domain | nindent 4 }} + + logger.yaml: | {{ toYaml .Values.configmap.logger | nindent 4 }} + + remote_data.yaml: | {{ toYaml .Values.configmap.remote_data | nindent 4 }} + + server.yaml: | {{ toYaml .Values.configmap.server | nindent 4 }} + + storage.yaml: | {{ toYaml .Values.configmap.storage | nindent 4 }} + + task_resource_defaults.yaml: | {{ toYaml .Values.configmap.task_resource_defaults | nindent 4 }} diff --git a/helm/templates/admin/cronjob.yaml b/helm/templates/admin/cronjob.yaml new file mode 100644 index 0000000000..68477543ed --- /dev/null +++ b/helm/templates/admin/cronjob.yaml @@ -0,0 +1,41 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: syncresources + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} +spec: + schedule: '*/1 * * * *' + jobTemplate: + spec: + template: + {{- with .Values.flyteadmin.podAnnotations }} + metadata: + annotations: {{ toYaml . | nindent 12 }} + {{- end }} + spec: + containers: + - command: + - flyteadmin + - --logtostderr + - --config + - {{ .Values.flyteadmin.configPath }} + - clusterresource + - sync + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + name: sync-cluster-resources + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 12 }} + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: OnFailure + serviceAccountName: flyteadmin + volumes: {{- include "databaseSecret.volume" . | nindent 10 }} + - configMap: + name: clusterresource-template + name: resource-templates + - configMap: + name: flyte-admin-config + name: config-volume diff --git a/helm/templates/admin/deployment.yaml b/helm/templates/admin/deployment.yaml new file mode 100644 index 0000000000..075488534a --- /dev/null +++ b/helm/templates/admin/deployment.yaml @@ -0,0 +1,133 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "flyteadmin.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.flyteadmin.replicaCount }} + selector: + matchLabels: {{ include "flyteadmin.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + configChecksum: {{ include (print .Template.BasePath "/admin/configmap.yaml") . | sha256sum | trunc 63 | quote }} + {{- with .Values.flyteadmin.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "flyteadmin.labels" . | nindent 8 }} + spec: + initContainers: + {{- if .Values.postgres.enabled }} + - name: check-db-ready + image: postgres:10.1 + command: + - sh + - -c + - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done; + {{- end }} + - command: + - flyteadmin + - --logtostderr + - --config + - {{ .Values.flyteadmin.configPath }} + - migrate + - run + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + name: run-migrations + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --logtostderr + - --config + - {{ .Values.flyteadmin.configPath }} + - migrate + - seed-projects + - flytesnacks + - flytetester + - flyteexamples + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + name: seed-projects + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --logtostderr + - --config + - {{ .Values.flyteadmin.configPath }} + - clusterresource + - sync + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + name: sync-cluster-resources + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + containers: + - command: + - flyteadmin + - --logtostderr + - --config + - {{ .Values.flyteadmin.configPath }} + {{- with .Values.flyteadmin.extraArgs }} + {{- toYaml . | nindent 8 }} + {{- end }} + - serve + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + name: flyteadmin + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: {{ toYaml .Values.flyteadmin.resources | nindent 10 }} + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /srv/flyte + name: shared-data + - mountPath: /etc/flyte/config + name: config-volume + - command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: Flyte Admin OpenAPI + - name: SPEC_URL + value: /api/v1/openapi + - name: PORT + value: "8087" + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + name: redoc + ports: + - containerPort: 8087 + resources: + limits: + cpu: "0.1" + memory: 200Mi + serviceAccountName: {{ template "flyteadmin.name" . }} + volumes: {{- include "databaseSecret.volume" . | nindent 6 }} + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-admin-config + name: config-volume + - configMap: + name: clusterresource-template + name: resource-templates + {{- with .Values.flyteadmin.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flyteadmin.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flyteadmin.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/admin/rbac.yaml b/helm/templates/admin/rbac.yaml new file mode 100644 index 0000000000..8b430aca9c --- /dev/null +++ b/helm/templates/admin/rbac.yaml @@ -0,0 +1,52 @@ + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "flyteadmin.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} + {{- with .Values.flyteadmin.serviceAccountAnnotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "flyteadmin.name" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + - flyte.lyft.com + - rbac.authorization.k8s.io + resources: + - configmaps + - flyteworkflows + - namespaces + - pods + - resourcequotas + - roles + - rolebindings + - secrets + - services + - serviceaccounts + - spark-role + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "flyteadmin.name" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "flyteadmin.name" . }} +subjects: +- kind: ServiceAccount + name: {{ template "flyteadmin.name" . }} + namespace: {{ template "flyte.namespace" . }} diff --git a/helm/templates/admin/service.yaml b/helm/templates/admin/service.yaml new file mode 100644 index 0000000000..2a2193940f --- /dev/null +++ b/helm/templates/admin/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "flyteadmin.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} + {{- with .Values.flyteadmin.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.flyteadmin.service.type}} + type: {{ . }} + {{- end }} + ports: + - name: redoc + port: 87 + protocol: TCP + targetPort: 8087 + - name: http + port: 80 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 81 + protocol: TCP + targetPort: 8089 + selector: {{ include "flyteadmin.selectorLabels" . | nindent 4 }} diff --git a/helm/templates/common/configmap.yaml b/helm/templates/common/configmap.yaml new file mode 100644 index 0000000000..606585656f --- /dev/null +++ b/helm/templates/common/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: clusterresource-template + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} +data: +{{- with .Values.configmap.aa_namespace }} + aa_namespace.yaml: | {{ . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.ab_project_resource_quota }} + ab_project-resource-quota.yaml: | {{ . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.ac_project_copilot_dataconfig }} + ac_project-copilot-dataconfig.yaml: | {{ . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.ad_spark_role }} + ad_spark-role.yaml: | {{ . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.ae_spark_service_account }} + ae_spark-service-account.yaml: | {{ . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.af_spark_role_binding }} + af_spark-role-binding.yaml: | {{ . | nindent 4 }} +{{- end }} diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml new file mode 100644 index 0000000000..93c65bcbcf --- /dev/null +++ b/helm/templates/common/ingress.yaml @@ -0,0 +1,40 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "flyte.name" . }} + namespace: {{ template "flyte.namespace" . }} + {{- with .Values.common.ingressAnnotations }} + annotations: {{ toYaml . | nindent 4}} + {{- end }} +spec: + rules: + - http: + paths: + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /console + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /__webpack_hmr + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /api + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /healthcheck + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /v1 + - backend: + serviceName: flyteadmin + servicePort: 81 + path: /flyteidl.service.AdminService + - backend: + serviceName: flyteadmin + servicePort: 87 + path: /openapi diff --git a/helm/templates/common/namespace.yaml b/helm/templates/common/namespace.yaml new file mode 100644 index 0000000000..bf158e57c9 --- /dev/null +++ b/helm/templates/common/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ template "flyte.namespace" . }} diff --git a/helm/templates/common/secret.yaml b/helm/templates/common/secret.yaml new file mode 100644 index 0000000000..7eb925de80 --- /dev/null +++ b/helm/templates/common/secret.yaml @@ -0,0 +1,4 @@ +{{- with .Values.common.databaseSecret.secretManifest }} +{{ toYaml . }} +{{- end }} + \ No newline at end of file diff --git a/helm/templates/console/configmap.yaml b/helm/templates/console/configmap.yaml new file mode 100644 index 0000000000..084091b781 --- /dev/null +++ b/helm/templates/console/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-console-config + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteconsole.labels" . | nindent 4 }} +data: {{ toYaml .Values.configmap.console | nindent 2 }} diff --git a/helm/templates/console/deployment.yaml b/helm/templates/console/deployment.yaml new file mode 100644 index 0000000000..298fb2661e --- /dev/null +++ b/helm/templates/console/deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "flyteconsole.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteconsole.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.flyteconsole.replicaCount }} + selector: + matchLabels: {{ include "flyteconsole.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + configChecksum: {{ include (print .Template.BasePath "/console/configmap.yaml") . | sha256sum | trunc 63 | quote }} + {{- with .Values.flyteconsole.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "flyteconsole.labels" . | nindent 8 }} + spec: + containers: + - image: "{{ .Values.flyteconsole.image.repository }}:{{ .Values.flyteconsole.image.tag }}" + imagePullPolicy: "{{ .Values.flyteconsole.image.pullPolicy }}" + name: flyteconsole + envFrom: + - configMapRef: + name: flyte-console-config + ports: + - containerPort: 8080 + resources: {{ toYaml .Values.flyteconsole.resources | nindent 10 }} + volumeMounts: + - mountPath: /srv/flyte + name: shared-data + volumes: + - emptyDir: {} + name: shared-data + {{- with .Values.flyteconsole.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flyteconsole.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flyteconsole.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/helm/templates/console/service.yaml b/helm/templates/console/service.yaml new file mode 100644 index 0000000000..c007243455 --- /dev/null +++ b/helm/templates/console/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "flyteconsole.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteconsole.labels" . | nindent 4 }} + {{- with .Values.flyteconsole.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.flyteconsole.service.type}} + type: {{ . }} + {{- end }} + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: {{ include "flyteconsole.selectorLabels" . | nindent 4 }} diff --git a/helm/templates/contour/crds/ingressroutes.yaml b/helm/templates/contour/crds/ingressroutes.yaml new file mode 100644 index 0000000000..d4b258c836 --- /dev/null +++ b/helm/templates/contour/crds/ingressroutes.yaml @@ -0,0 +1,136 @@ +{{- if .Values.contour.enabled }} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressroutes.contour.heptio.com + labels: + component: ingressroute +spec: + group: contour.heptio.com + version: v1beta1 + scope: Namespaced + names: + plural: ingressroutes + kind: IngressRoute + additionalPrinterColumns: + - name: FQDN + type: string + description: Fully qualified domain name + JSONPath: .spec.virtualhost.fqdn + - name: TLS Secret + type: string + description: Secret with TLS credentials + JSONPath: .spec.virtualhost.tls.secretName + - name: First route + type: string + description: First routes defined + JSONPath: .spec.routes[0].match + - name: Status + type: string + description: The current status of the IngressRoute + JSONPath: .status.currentStatus + - name: Status Description + type: string + description: Description of the current status + JSONPath: .status.description + validation: + openAPIV3Schema: + properties: + spec: + required: + - routes + properties: + virtualhost: + properties: + fqdn: + type: string + pattern: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-z]{2,}$ + aliases: + type: array + items: + type: string + pattern: ^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$ + strategy: + type: string + enum: + - RoundRobin + - LeastRequest + - Random + - RingHash + - Maglev + healthCheck: + type: object + required: + - path + properties: + path: + type: string + pattern: ^\/.*$ + intervalSeconds: + type: integer + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + healthyThresholdCount: + type: integer + routes: + type: array + items: + required: + - match + properties: + match: + type: string + pattern: ^\/.*$ + delegate: + type: object + required: + - name + properties: + name: + type: string + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 + namespace: + type: string + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 + services: + type: array + items: + type: object + required: + - name + - port + properties: + name: + type: string + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 + port: + type: integer + weight: + type: integer + strategy: + type: string + enum: + - RoundRobin + - WeightedLeastRequest + - Random + - RingHash + - Maglev + healthCheck: + type: object + required: + - path + properties: + path: + type: string + pattern: ^\/.*$ + intervalSeconds: + type: integer + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + healthyThresholdCount: + type: integer +{{- end }} diff --git a/helm/templates/contour/deployment.yaml b/helm/templates/contour/deployment.yaml new file mode 100644 index 0000000000..d4733d7257 --- /dev/null +++ b/helm/templates/contour/deployment.yaml @@ -0,0 +1,83 @@ +{{- if .Values.contour.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "contour.name" . }} + namespace: {{ template "contour.namespace" . }} + labels: {{ include "contour.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.contour.replicaCount }} + selector: + matchLabels: {{ include "contour.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.contour.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "contour.labels" . | nindent 8 }} + spec: + initContainers: + - image: "{{ .Values.contour.contour.image.repository }}:{{ .Values.contour.contour.image.tag }}" + imagePullPolicy: "{{ .Values.contour.contour.image.pullPolicy }}" + name: envoy-initconfig + command: + - contour + args: + - bootstrap + - /config/contour.yaml + - --statsd-enabled + volumeMounts: + - name: contour-config + mountPath: /config + containers: + - image: "{{ .Values.contour.envoy.image.repository }}:{{ .Values.contour.envoy.image.tag }}" + imagePullPolicy: "{{ .Values.contour.envoy.image.pullPolicy }}" + name: envoy + command: + - envoy + args: + - -c + - /config/contour.yaml + - --service-cluster + - cluster0 + - --service-node + - node0 + ports: + - containerPort: 80 + name: http + - containerPort: 8002 + name: statsd + resources: {{ toYaml .Values.contour.envoy.resources | nindent 10 }} + volumeMounts: + - name: contour-config + mountPath: /config + - image: "{{ .Values.contour.contour.image.repository }}:{{ .Values.contour.contour.image.tag }}" + imagePullPolicy: "{{ .Values.contour.contour.image.pullPolicy }}" + ports: + - containerPort: 8000 + name: contour + name: contour + command: + - contour + args: + - serve + - --incluster + - --envoy-http-port=80 + - --debug-http-port=6069 + resources: {{ toYaml .Values.contour.contour.resources | nindent 10 }} + volumes: + - name: contour-config + emptyDir: {} + dnsPolicy: ClusterFirst + serviceAccountName: {{ template "contour.name" . }} + terminationGracePeriodSeconds: 30 + {{- with .Values.contour.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.contour.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.contour.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/templates/contour/namespace.yaml b/helm/templates/contour/namespace.yaml new file mode 100644 index 0000000000..5262f4fb84 --- /dev/null +++ b/helm/templates/contour/namespace.yaml @@ -0,0 +1,6 @@ +{{- if .Values.sparkoperator.enabled }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ template "contour.namespace" . }} +{{- end }} diff --git a/helm/templates/contour/rbac.yaml b/helm/templates/contour/rbac.yaml new file mode 100644 index 0000000000..00d6d43868 --- /dev/null +++ b/helm/templates/contour/rbac.yaml @@ -0,0 +1,78 @@ +{{- if .Values.contour.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "contour.name" . }} + namespace: {{ template "contour.namespace" . }} + labels: {{ include "contour.labels" . | nindent 4 }} + {{- with .Values.contour.serviceAccountAnnotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "contour.name" . }} + labels: {{ include "contour.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: ["contour.heptio.com"] + resources: ["ingressroutes"] + verbs: + - get + - list + - watch + - put + - post + - patch + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "contour.name" . }} + labels: {{ include "contour.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "contour.name" . }} +subjects: +- kind: ServiceAccount + name: {{ template "contour.name" . }} + namespace: {{ template "contour.namespace" . }} + +{{- end }} diff --git a/helm/templates/contour/service.yaml b/helm/templates/contour/service.yaml new file mode 100644 index 0000000000..e64e19d6ff --- /dev/null +++ b/helm/templates/contour/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.contour.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "contour.name" . }} + namespace: {{ template "contour.namespace" . }} + labels: {{ include "contour.labels" . | nindent 4 }} + {{- with .Values.contour.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.contour.service.type}} + type: {{ . }} + {{- end }} + ports: {{ toYaml .Values.contour.service.ports | nindent 4 }} + selector: {{ include "contour.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/templates/datacatalog/configmap.yaml b/helm/templates/datacatalog/configmap.yaml new file mode 100644 index 0000000000..a552b02dac --- /dev/null +++ b/helm/templates/datacatalog/configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: datacatalog-config + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "datacatalog.labels" . | nindent 4 }} +data: + db.yaml: | {{ toYaml .Values.configmap.db | nindent 4 }} + + logger.yaml: | {{ toYaml .Values.configmap.logger | nindent 4 }} + + server.yaml: | {{ toYaml .Values.configmap.datacatalogServer | nindent 4 }} + + storage.yaml: | {{ toYaml .Values.configmap.storage | nindent 4 }} diff --git a/helm/templates/datacatalog/deployment.yaml b/helm/templates/datacatalog/deployment.yaml new file mode 100644 index 0000000000..23981b1c0a --- /dev/null +++ b/helm/templates/datacatalog/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "datacatalog.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "datacatalog.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.datacatalog.replicaCount }} + selector: + matchLabels: {{ include "datacatalog.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + configChecksum: {{ include (print .Template.BasePath "/datacatalog/configmap.yaml") . | sha256sum | trunc 63 | quote }} + {{- with .Values.datacatalog.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "datacatalog.labels" . | nindent 8 }} + spec: + initContainers: + - command: + - datacatalog + - --logtostderr + - --config + - {{ .Values.datacatalog.configPath }} + - migrate + - run + image: "{{ .Values.datacatalog.image.repository }}:{{ .Values.datacatalog.image.tag }}" + imagePullPolicy: "{{ .Values.datacatalog.image.pullPolicy }}" + name: run-migrations + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/datacatalog/config + name: config-volume + containers: + - command: + - datacatalog + - --logtostderr + - --config + - {{ .Values.datacatalog.configPath }} + {{- with .Values.datacatalog.extraArgs }} + {{- toYaml . | nindent 8 }} + {{- end }} + - serve + image: "{{ .Values.datacatalog.image.repository }}:{{ .Values.datacatalog.image.tag }}" + imagePullPolicy: "{{ .Values.datacatalog.image.pullPolicy }}" + name: datacatalog + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: {{ toYaml .Values.datacatalog.resources | nindent 10 }} + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/datacatalog/config + name: config-volume + serviceAccountName: {{ template "datacatalog.name" . }} + volumes: {{- include "databaseSecret.volume" . | nindent 6 }} + - emptyDir: {} + name: shared-data + - configMap: + name: datacatalog-config + name: config-volume + {{- with .Values.datacatalog.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.datacatalog.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.datacatalog.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/datacatalog/rbac.yaml b/helm/templates/datacatalog/rbac.yaml new file mode 100644 index 0000000000..ca1ab89afd --- /dev/null +++ b/helm/templates/datacatalog/rbac.yaml @@ -0,0 +1,11 @@ + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "datacatalog.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "datacatalog.labels" . | nindent 4 }} + {{- with .Values.datacatalog.serviceAccountAnnotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} diff --git a/helm/templates/datacatalog/service.yaml b/helm/templates/datacatalog/service.yaml new file mode 100644 index 0000000000..07958d2f34 --- /dev/null +++ b/helm/templates/datacatalog/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "datacatalog.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "datacatalog.labels" . | nindent 4 }} + {{- with .Values.datacatalog.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.datacatalog.service.type}} + type: {{ . }} + {{- end }} + ports: + - name: grpc-2 + port: 8089 + protocol: TCP + targetPort: 8089 + - name: http + port: 88 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 89 + protocol: TCP + targetPort: 8089 + selector: {{ include "datacatalog.selectorLabels" . | nindent 4 }} diff --git a/helm/templates/minio/deployment.yaml b/helm/templates/minio/deployment.yaml new file mode 100644 index 0000000000..63b57d293c --- /dev/null +++ b/helm/templates/minio/deployment.yaml @@ -0,0 +1,50 @@ +{{- if .Values.minio.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "minio.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "minio.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.minio.replicaCount }} + selector: + matchLabels: {{ include "minio.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.minio.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "minio.labels" . | nindent 8 }} + spec: + containers: + - image: "{{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}" + imagePullPolicy: "{{ .Values.minio.image.pullPolicy }}" + name: minio + args: + - server + - /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: miniostorage + ports: + - containerPort: 9000 + name: minio + resources: {{ toYaml .Values.minio.resources | nindent 10 }} + volumeMounts: + - name: minio-storage + mountPath: /var/lib/minioql/data + volumes: + - name: minio-storage + emptyDir: {} + {{- with .Values.minio.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.minio.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.minio.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/templates/minio/service.yaml b/helm/templates/minio/service.yaml new file mode 100644 index 0000000000..b9b56cc1bc --- /dev/null +++ b/helm/templates/minio/service.yaml @@ -0,0 +1,21 @@ +{{- if .Values.minio.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "minio.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "minio.labels" . | nindent 4 }} + {{- with .Values.minio.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.minio.service.type}} + type: {{ . }} + {{- end }} + ports: + - name: minio + port: 9000 + protocol: TCP + targetPort: minio + selector: {{ include "minio.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/templates/postgres/deployment.yaml b/helm/templates/postgres/deployment.yaml new file mode 100644 index 0000000000..13ec8209c3 --- /dev/null +++ b/helm/templates/postgres/deployment.yaml @@ -0,0 +1,42 @@ +{{- if .Values.postgres.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "postgres.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "postgres.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.postgres.replicaCount }} + selector: + matchLabels: {{ include "postgres.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.postgres.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "postgres.labels" . | nindent 8 }} + spec: + containers: + - image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}" + imagePullPolicy: "{{ .Values.postgres.image.pullPolicy }}" + name: postgres + ports: + - containerPort: 5432 + name: postgres + resources: {{ toYaml .Values.postgres.resources | nindent 10 }} + volumeMounts: + - name: postgres-storage + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-storage + emptyDir: {} + {{- with .Values.postgres.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.postgres.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.postgres.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/templates/postgres/service.yaml b/helm/templates/postgres/service.yaml new file mode 100644 index 0000000000..4309df215e --- /dev/null +++ b/helm/templates/postgres/service.yaml @@ -0,0 +1,21 @@ +{{- if .Values.minio.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "postgres.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "postgres.labels" . | nindent 4 }} + {{- with .Values.postgres.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.postgres.service.type}} + type: {{ . }} + {{- end }} + ports: + - name: postgres + port: 5432 + protocol: TCP + targetPort: postgres + selector: {{ include "postgres.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/templates/propeller/configmap.yaml b/helm/templates/propeller/configmap.yaml new file mode 100644 index 0000000000..5d341c05bd --- /dev/null +++ b/helm/templates/propeller/configmap.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-propeller-config + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} +data: + admin.yaml: | {{ toYaml .Values.configmap.admin | nindent 4 }} + + catalog.yaml: | {{ toYaml .Values.configmap.catalog | nindent 4 }} + + catalog_cache.yaml: | {{ toYaml .Values.configmap.catalog_cache | nindent 4 }} + + copilot.yaml: | {{ toYaml .Values.configmap.copilot | nindent 4 }} + + core.yaml: | {{ toYaml .Values.configmap.core | nindent 4 }} + + enabled_plugins.yaml: | {{ toYaml .Values.configmap.enabled_plugins | nindent 4 }} + + k8s.yaml: | {{ toYaml .Values.configmap.k8s | nindent 4 }} + + logger.yaml: | {{ toYaml .Values.configmap.logger | nindent 4 }} + + qubole.yaml: | {{ toYaml .Values.configmap.qubole | nindent 4 }} + + resource_manager.yaml: | {{ toYaml .Values.configmap.resource_manager | nindent 4 }} + + spark.yaml: | {{ toYaml .Values.configmap.spark | nindent 4 }} + + storage.yaml: | {{ toYaml .Values.configmap.storage | nindent 4 }} + + task_logs.yaml: | {{ toYaml .Values.configmap.task_logs | nindent 4 }} diff --git a/helm/templates/propeller/crds/flyteworkflow.yaml b/helm/templates/propeller/crds/flyteworkflow.yaml new file mode 100644 index 0000000000..1095feca6b --- /dev/null +++ b/helm/templates/propeller/crds/flyteworkflow.yaml @@ -0,0 +1,14 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: flyteworkflows.flyte.lyft.com +spec: + group: flyte.lyft.com + names: + kind: FlyteWorkflow + plural: flyteworkflows + shortNames: + - fly + singular: flyteworkflow + scope: Namespaced + version: v1alpha1 diff --git a/helm/templates/propeller/deployment.yaml b/helm/templates/propeller/deployment.yaml new file mode 100644 index 0000000000..fc11b1eb3d --- /dev/null +++ b/helm/templates/propeller/deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "flytepropeller.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flytepropeller.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.flytepropeller.replicaCount }} + selector: + matchLabels: {{ include "flytepropeller.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + configChecksum: {{ include (print .Template.BasePath "/propeller/configmap.yaml") . | sha256sum | trunc 63 | quote }} + {{- with .Values.flytepropeller.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "flytepropeller.labels" . | nindent 8 }} + spec: + containers: + - command: + - flytepropeller + - --config + - {{ .Values.flytepropeller.configPath }} + {{- with .Values.flytepropeller.extraArgs }} + {{- toYaml . | nindent 8 }} + {{- end }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: "{{ .Values.flytepropeller.image.repository }}:{{ .Values.flytepropeller.image.tag }}" + imagePullPolicy: "{{ .Values.flytepropeller.image.pullPolicy }}" + name: flytepropeller + ports: + - containerPort: 10254 + resources: {{ toYaml .Values.flytepropeller.resources | nindent 10 }} + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + serviceAccountName: {{ template "flytepropeller.name" . }} + volumes: + - configMap: + name: flyte-propeller-config + name: config-volume + {{- with .Values.flytepropeller.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flytepropeller.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flytepropeller.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/propeller/rbac.yaml b/helm/templates/propeller/rbac.yaml new file mode 100644 index 0000000000..dd0cee2788 --- /dev/null +++ b/helm/templates/propeller/rbac.yaml @@ -0,0 +1,88 @@ + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "flytepropeller.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flytepropeller.labels" . | nindent 4 }} + {{- with .Values.flytepropeller.serviceAccountAnnotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "flytepropeller.name" . }} + labels: {{ include "flytepropeller.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - delete + - patch +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - flyte.lyft.com + resources: + - flyteworkflows + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - post + - deletecollection + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "flytepropeller.name" . }} + labels: {{ include "flytepropeller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "flytepropeller.name" . }} +subjects: +- kind: ServiceAccount + name: {{ template "flytepropeller.name" . }} + namespace: {{ template "flyte.namespace" . }} diff --git a/helm/templates/pytorch-operator/crds/pytorchjobs.yaml b/helm/templates/pytorch-operator/crds/pytorchjobs.yaml new file mode 100644 index 0000000000..ae2b7b7e78 --- /dev/null +++ b/helm/templates/pytorch-operator/crds/pytorchjobs.yaml @@ -0,0 +1,44 @@ +{{- if .Values.pytorchoperator.enabled }} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pytorchjobs.kubeflow.org +spec: + additionalPrinterColumns: + - JSONPath: .status.conditions[-1:].type + name: State + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: kubeflow.org + names: + kind: PyTorchJob + plural: pytorchjobs + singular: pytorchjob + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + pytorchReplicaSpecs: + properties: + Master: + properties: + replicas: + maximum: 1 + minimum: 1 + type: integer + Worker: + properties: + replicas: + minimum: 1 + type: integer + versions: + - name: v1 + served: true + storage: true +{{- end }} diff --git a/helm/templates/pytorch-operator/deployment.yaml b/helm/templates/pytorch-operator/deployment.yaml new file mode 100644 index 0000000000..02dc98fb01 --- /dev/null +++ b/helm/templates/pytorch-operator/deployment.yaml @@ -0,0 +1,48 @@ +{{- if .Values.pytorchoperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pytorch-operator.name" . }} + namespace: {{ template "pytorch-operator.namespace" . }} + labels: {{ include "pytorch-operator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.pytorchoperator.replicaCount }} + selector: + matchLabels: {{ include "pytorch-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.pytorchoperator.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "pytorch-operator.labels" . | nindent 8 }} + spec: + containers: + - image: "{{ .Values.pytorchoperator.image.repository }}:{{ .Values.pytorchoperator.image.tag }}" + imagePullPolicy: "{{ .Values.pytorchoperator.image.pullPolicy }}" + name: pytorch-operator + command: + - /pytorch-operator.v1 + - --alsologtostderr + - -v=1 + - --monitoring-port=8443 + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + resources: {{ toYaml .Values.pytorchoperator.resources | nindent 10 }} + serviceAccountName: {{ template "pytorch-operator.name" . }} + {{- with .Values.pytorchoperator.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pytorchoperator.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pytorchoperator.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/templates/pytorch-operator/namespace.yaml b/helm/templates/pytorch-operator/namespace.yaml new file mode 100644 index 0000000000..6e15ce3dc0 --- /dev/null +++ b/helm/templates/pytorch-operator/namespace.yaml @@ -0,0 +1,6 @@ +{{- if .Values.pytorchoperator.enabled }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ template "pytorch-operator.namespace" . }} +{{- end }} diff --git a/helm/templates/pytorch-operator/rbac.yaml b/helm/templates/pytorch-operator/rbac.yaml new file mode 100644 index 0000000000..fe980a9b87 --- /dev/null +++ b/helm/templates/pytorch-operator/rbac.yaml @@ -0,0 +1,108 @@ +{{- if .Values.pytorchoperator.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pytorch-operator.name" . }} + namespace: {{ template "pytorch-operator.namespace" . }} + labels: {{ include "pytorch-operator.labels" . | nindent 4 }} + {{- with .Values.pytorchoperator.serviceAccountAnnotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pytorch-operator.name" . }}-admin + labels: {{ include "pytorch-operator.labels" . | nindent 4 }} +rules: [] +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pytorchjobs-admin: "true" + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pytorch-operator.name" . }}-edit + labels: {{ include "pytorch-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pytorch-operator.name" . }}-view + labels: {{ include "pytorch-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - get + - list + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pytorch-operator.name" . }} + labels: {{ include "pytorch-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - services + - endpoints + - events + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pytorch-operator.name" . }} + labels: {{ include "pytorch-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pytorch-operator.name" . }} +subjects: +- kind: ServiceAccount + name: {{ template "pytorch-operator.name" . }} + namespace: {{ template "pytorch-operator.namespace" . }} +{{- end }} diff --git a/helm/templates/pytorch-operator/service.yaml b/helm/templates/pytorch-operator/service.yaml new file mode 100644 index 0000000000..3deea516e7 --- /dev/null +++ b/helm/templates/pytorch-operator/service.yaml @@ -0,0 +1,20 @@ +{{- if .Values.pytorchoperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pytorch-operator.name" . }} + namespace: {{ template "pytorch-operator.namespace" . }} + labels: {{ include "pytorch-operator.labels" . | nindent 4 }} + {{- with .Values.pytorchoperator.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.pytorchoperator.service.type}} + type: {{ . }} + {{- end }} + ports: + - name: monitoring-port + port: 8443 + targetPort: 8443 + selector: {{ include "pytorch-operator.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/templates/redis/service.yaml b/helm/templates/redis/service.yaml new file mode 100644 index 0000000000..46afc98df5 --- /dev/null +++ b/helm/templates/redis/service.yaml @@ -0,0 +1,21 @@ +{{- if .Values.redis.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "redis.name" . }}-resource-manager + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "redis.labels" . | nindent 4 }} + {{- with .Values.redis.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.redis.service.type}} + type: {{ . }} + {{- end }} + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: {{ include "redis.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/templates/redis/statefulset.yaml b/helm/templates/redis/statefulset.yaml new file mode 100644 index 0000000000..93062efb22 --- /dev/null +++ b/helm/templates/redis/statefulset.yaml @@ -0,0 +1,70 @@ +{{- if .Values.redis.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "redis.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "redis.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.redis.replicaCount }} + selector: + matchLabels: {{ include "redis.selectorLabels" . | nindent 6 }} + serviceName: {{ template "redis.name" . }}-resource-manager + template: + metadata: + {{- with .Values.redis.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "redis.labels" . | nindent 8 }} + spec: + containers: + - image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" + imagePullPolicy: "{{ .Values.redis.image.pullPolicy }}" + name: redis + env: + - name: REDIS_PASSWORD + value: mypassword + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - containerPort: 6379 + name: redis + protocol: TCP + resources: {{ toYaml .Values.redis.resources | nindent 10 }} + volumeMounts: + - mountPath: /bitnami + name: redis-data + + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data + {{- with .Values.redis.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.redis.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.redis.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/templates/sparkoperator/crds/scheduledsparkapplications.yaml b/helm/templates/sparkoperator/crds/scheduledsparkapplications.yaml new file mode 100644 index 0000000000..6df8778411 --- /dev/null +++ b/helm/templates/sparkoperator/crds/scheduledsparkapplications.yaml @@ -0,0 +1,103 @@ +{{- if .Values.sparkoperator.enabled }} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: scheduledsparkapplications.sparkoperator.k8s.io +spec: + group: sparkoperator.k8s.io + names: + kind: ScheduledSparkApplication + listKind: ScheduledSparkApplicationList + plural: scheduledsparkapplications + shortNames: + - scheduledsparkapp + singular: scheduledsparkapplication + scope: Namespaced + validation: + openAPIV3Schema: + properties: + spec: + properties: + concurrencyPolicy: + enum: + - Allow + - Forbid + - Replace + failedRunHistoryLimit: + minimum: 1 + type: integer + schedule: + type: string + successfulRunHistoryLimit: + minimum: 1 + type: integer + template: + properties: + deps: + properties: + downloadTimeout: + minimum: 1 + type: integer + maxSimultaneousDownloads: + minimum: 1 + type: integer + driver: + properties: + cores: + exclusiveMinimum: true + minimum: 0 + type: number + podName: + pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' + executor: + properties: + cores: + exclusiveMinimum: true + minimum: 0 + type: number + instances: + minimum: 1 + type: integer + mode: + enum: + - cluster + - client + monitoring: + properties: + prometheus: + properties: + port: + maximum: 49151 + minimum: 1024 + type: integer + pythonVersion: + enum: + - "2" + - "3" + restartPolicy: + properties: + onFailureRetries: + minimum: 0 + type: integer + onFailureRetryInterval: + minimum: 1 + type: integer + onSubmissionFailureRetries: + minimum: 0 + type: integer + onSubmissionFailureRetryInterval: + minimum: 1 + type: integer + type: + enum: + - Never + - OnFailure + - Always + type: + enum: + - Java + - Scala + - Python + - R + version: v1beta1 +{{- end }} diff --git a/helm/templates/sparkoperator/crds/sparkapplications.yaml b/helm/templates/sparkoperator/crds/sparkapplications.yaml new file mode 100644 index 0000000000..c4bed90488 --- /dev/null +++ b/helm/templates/sparkoperator/crds/sparkapplications.yaml @@ -0,0 +1,149 @@ +{{- if .Values.sparkoperator.enabled }} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: sparkapplications.sparkoperator.k8s.io +spec: + group: sparkoperator.k8s.io + names: + kind: SparkApplication + listKind: SparkApplicationList + plural: sparkapplications + shortNames: + - sparkapp + singular: sparkapplication + scope: Namespaced + validation: + openAPIV3Schema: + properties: + metadata: + properties: + name: + maxLength: 63 + minLength: 1 + type: string + spec: + properties: + arguments: + items: + type: string + type: array + deps: + properties: + downloadTimeout: + minimum: 1 + type: integer + maxSimultaneousDownloads: + minimum: 1 + type: integer + driver: + properties: + cores: + exclusiveMinimum: true + minimum: 0 + type: number + podName: + pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' + executor: + properties: + cores: + exclusiveMinimum: true + minimum: 0 + type: number + instances: + minimum: 1 + type: integer + failureRetries: + type: integer + hadoopConf: + type: object + hadoopConfigMap: + type: string + image: + type: string + imagePullPolicy: + enum: + - Always + - Never + - IfNotPresent + imagePullSecrets: + items: + type: string + type: array + initContainerImage: + type: string + mainApplicationFile: + type: string + mainClass: + type: string + memoryOverheadFactor: + type: string + mode: + enum: + - cluster + - client + monitoring: + properties: + exposeDriverMetrics: + type: boolean + exposeExecutorMetrics: + type: boolean + metricsProperties: + type: string + prometheus: + properties: + port: + maximum: 49151 + minimum: 1024 + type: integer + nodeSelector: + type: object + pythonVersion: + enum: + - "2" + - "3" + restartPolicy: + properties: + onFailureRetries: + minimum: 0 + type: integer + onFailureRetryInterval: + minimum: 1 + type: integer + onSubmissionFailureRetries: + minimum: 0 + type: integer + onSubmissionFailureRetryInterval: + minimum: 1 + type: integer + type: + enum: + - Never + - OnFailure + - Always + retryInterval: + type: integer + sparkConf: + type: object + sparkConfigMap: + type: string + sparkVersion: + type: string + type: + enum: + - Java + - Scala + - Python + - R + volumes: + items: + properties: + name: + type: string + type: object + type: array + required: + - type + - sparkVersion + version: v1beta1 +{{- end }} diff --git a/helm/templates/sparkoperator/deployment.yaml b/helm/templates/sparkoperator/deployment.yaml new file mode 100644 index 0000000000..07734eb29d --- /dev/null +++ b/helm/templates/sparkoperator/deployment.yaml @@ -0,0 +1,59 @@ +{{- if .Values.sparkoperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "sparkoperator.name" . }} + namespace: {{ template "sparkoperator.namespace" . }} + labels: {{ include "sparkoperator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.sparkoperator.replicaCount }} + selector: + matchLabels: {{ include "sparkoperator.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.sparkoperator.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "sparkoperator.labels" . | nindent 8 }} + spec: + containers: + - image: "{{ .Values.sparkoperator.image.repository }}:{{ .Values.sparkoperator.image.tag }}" + imagePullPolicy: "{{ .Values.sparkoperator.image.pullPolicy }}" + name: sparkoperator + command: + - /usr/bin/spark-operator + args: + - -logtostderr + - -v=2 + - -controller-threads=20 + - -enable-metrics=true + - "-metrics-prefix=service:" + - -metrics-labels=task_name + - -metrics-labels=workflow_name + - -enable-webhook=true + - -webhook-svc-namespace=sparkoperator + env: + - name: GOMAXPROCS + value: "16" + ports: + - containerPort: 10254 + - containerPort: 8080 + resources: {{ toYaml .Values.sparkoperator.resources | nindent 10 }} + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook-certs + serviceAccountName: {{ template "sparkoperator.name" . }} + volumes: + - name: webhook-certs + secret: + secretName: spark-webhook-certs + {{- with .Values.sparkoperator.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.sparkoperator.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.sparkoperator.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/templates/sparkoperator/job.yaml b/helm/templates/sparkoperator/job.yaml new file mode 100644 index 0000000000..fc155d54cb --- /dev/null +++ b/helm/templates/sparkoperator/job.yaml @@ -0,0 +1,28 @@ +{{- if .Values.sparkoperator.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "sparkoperator.name" . }}-init + namespace: {{ template "sparkoperator.namespace" . }} + labels: {{ include "sparkoperator.labels" . | nindent 4 }} +spec: + backoffLimit: 3 + template: + metadata: + {{- with .Values.sparkoperator.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "sparkoperator.labels" . | nindent 8 }} + spec: + containers: + - image: "{{ .Values.sparkoperator.image.repository }}:{{ .Values.sparkoperator.image.tag }}" + imagePullPolicy: "{{ .Values.sparkoperator.image.pullPolicy }}" + name: sparkoperator + command: + - /usr/bin/gencerts.sh + - --namespace + - sparkoperator + - -p + restartPolicy: Never + serviceAccountName: {{ template "sparkoperator.name" . }} +{{- end }} diff --git a/helm/templates/sparkoperator/namespace.yaml b/helm/templates/sparkoperator/namespace.yaml new file mode 100644 index 0000000000..b4b2ad7dda --- /dev/null +++ b/helm/templates/sparkoperator/namespace.yaml @@ -0,0 +1,6 @@ +{{- if .Values.sparkoperator.enabled }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ template "sparkoperator.namespace" . }} +{{- end }} diff --git a/helm/templates/sparkoperator/rbac.yaml b/helm/templates/sparkoperator/rbac.yaml new file mode 100644 index 0000000000..7a6e8c6083 --- /dev/null +++ b/helm/templates/sparkoperator/rbac.yaml @@ -0,0 +1,167 @@ +{{- if .Values.sparkoperator.enabled }} + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "sparkoperator.name" . }} + namespace: {{ template "sparkoperator.namespace" . }} + labels: {{ include "sparkoperator.labels" . | nindent 4 }} + {{- with .Values.sparkoperator.serviceAccountAnnotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "sparkoperator.name" . }} + labels: {{ include "sparkoperator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - resourcequotas + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + - secrets + verbs: + - create + - get + - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - create + - get + - update + - delete +- apiGroups: + - "" + resources: + - pods + verbs: + - '*' +- apiGroups: + - "" + resources: + - services + - configmaps + - secrets + verbs: + - create + - get + - delete +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - create + - get + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - get + - update + - delete + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - update + - delete +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - get + - update + - delete +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + - scheduledsparkapplications + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "sparkoperator.name" . }}-aggregate-to-admin + labels: {{ include "sparkoperator.labels" . | nindent 4 }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + - scheduledsparkapplications + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "sparkoperator.name" . }} + labels: {{ include "sparkoperator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "sparkoperator.name" . }} +subjects: +- kind: ServiceAccount + name: {{ template "sparkoperator.name" . }} + namespace: {{ template "sparkoperator.namespace" . }} + +{{- end }} diff --git a/helm/templates/sparkoperator/service.yaml b/helm/templates/sparkoperator/service.yaml new file mode 100644 index 0000000000..90840e8611 --- /dev/null +++ b/helm/templates/sparkoperator/service.yaml @@ -0,0 +1,20 @@ +{{- if .Values.sparkoperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "sparkoperator.name" . }} + namespace: {{ template "sparkoperator.namespace" . }} + labels: {{ include "sparkoperator.labels" . | nindent 4 }} + {{- with .Values.sparkoperator.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.sparkoperator.service.type}} + type: {{ . }} + {{- end }} + ports: + - port: 443 + targetPort: 8080 + name: webhook + selector: {{ include "sparkoperator.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml new file mode 100644 index 0000000000..d1a113a841 --- /dev/null +++ b/helm/values-eks.yaml @@ -0,0 +1,645 @@ + +# +# FLYTEADMIN +# + +flyteadmin: + replicaCount: 2 + image: + repository: docker.io/lyft/flyteadmin + tag: v0.3.7 + pullPolicy: IfNotPresent + + # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte + + resources: + limits: + cpu: 250m + ephemeral-storage: 200Mi + memory: 500Mi + requests: + cpu: 50m + ephemeral-storage: 200Mi + memory: 200Mi + + configPath: /etc/flyte/config/*.yaml + service: + annotations: + contour.heptio.com/upstream-protocol.h2c: grpc + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: flyteadmin + topologyKey: kubernetes.io/hostname + +# +# DATACATALOG +# + +datacatalog: + replicaCount: 2 + image: + repository: docker.io/lyft/datacatalog + tag: v0.2.2 + pullPolicy: IfNotPresent + + # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte + + resources: + limits: + cpu: 500m + ephemeral-storage: 200Mi + memory: 500Mi + requests: + cpu: 50m + ephemeral-storage: 200Mi + memory: 200Mi + + configPath: /etc/datacatalog/config/*.yaml + + service: + annotations: + contour.heptio.com/upstream-protocol.h2c: grpc + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: datacatalog + topologyKey: kubernetes.io/hostname + +# +# FLYTEPROPELLER +# + +flytepropeller: + replicaCount: 2 + image: + repository: docker.io/lyft/flytepropeller + tag: v0.4.2 + pullPolicy: IfNotPresent + + # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte + + resources: + limits: + cpu: 500m + ephemeral-storage: 200Mi + memory: 500Mi + requests: + cpu: 50m + ephemeral-storage: 200Mi + memory: 200Mi + configPath: /etc/flyte/config*/*.yaml + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: flytepropeller + topologyKey: kubernetes.io/hostname + +# +# FLYTECONSOLE +# + +flyteconsole: + replicaCount: 2 + image: + repository: docker.io/lyft/flyteconsole + tag: v0.12.1 + pullPolicy: IfNotPresent + resources: + limits: + cpu: 250m + memory: 250Mi + requests: + cpu: 10m + memory: 50Mi + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: flyteconsole + topologyKey: kubernetes.io/hostname + +# +# REDIS +# + +redis: + enabled: true + replicaCount: 1 + image: + repository: docker.io/bitnami/redis + tag: 4.0.2-r1 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 250Mi + +# +# POSTGRES +# + +postgres: + enabled: false # recommended adding managed RDS host to the config + +# +# MINIO +# + +minio: + enabled: false # Use AWS S3 bucket instead + +# +# CONTOUR +# + +contour: + enabled: true + replicaCount: 2 + contour: + image: + repository: gcr.io/heptio-images/contour + tag: v0.6.1 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 50m + memory: 100Mi + limits: + cpu: 2000m + memory: 1000Mi + envoy: + image: + repository: docker.io/envoyproxy/envoy-alpine + tag: v1.6.0 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 50m + memory: 100Mi + limits: + cpu: 2000m + memory: 1000Mi + + service: + annotations: + external-dns.alpha.kubernetes.io/hostname: flyte.examlple.com + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-east-1::certificate/" + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp" + service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: "*" + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "3600" + service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + service.beta.kubernetes.io/aws-load-balancer-type: nlb + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443" + type: LoadBalancer + ports: + - name: tcp + protocol: TCP + port: 80 + - name: tls + protocol: TCP + port: 443 + targetPort: 80 + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: contour + topologyKey: kubernetes.io/hostname + +# +# SPARKOPERATOR +# + +sparkoperator: + enabled: true + replicaCount: 1 + image: + repository: gcr.io/spark-operator/spark-operator + tag: v2.4.0-v1beta1-0.9.0 + pullPolicy: IfNotPresent + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 50m + memory: 250Mi + +# +# PYTORCHOPERATOR +# + +pytorchoperator: + enabled: true + replicaCount: 1 + image: + repository: gcr.io/kubeflow-images-public/pytorch-operator + tag: v1.0.0-g047cf0f + pullPolicy: IfNotPresent + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 50m + memory: 250Mi + +# +# COMMON +# + +common: + databaseSecret: + name: db-pass + secretManifest: + # # Leave it empty if your secret already exists + + # # Else you can create your own secret object: + # # Necessary dependencies: + # # - https://github.com/hashicorp/vault + # # - https://github.com/godaddy/kubernetes-external-secrets + + # apiVersion: kubernetes-client.io/v1 + # kind: ExternalSecret + # metadata: + # name: db-pass + # namespace: flyte + # spec: + # backendType: vault + # vaultMountPoint: + # vaultRole: + # data: + # - name: pass.txt + # key: k8s/data/path/to/secret + # property: password + +# +# CONFIGMAPS +# + +configmap: + + console: + BASE_URL: /console + CONFIG_DIR: /etc/flyte/config + + cluster_resources: + cluster_resources: + templatePath: "/etc/flyte/clusterresource/templates" + customData: + production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "3000Mi" + refresh: 5m + + db: + database: + port: 5432 + username: postgres + host: + dbname: flyte-development + passwordPath: /etc/db/pass.txt + + domain: + domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + + remote_data: + remoteData: + region: us-east-1 + scheme: aws + signedUrls: + durationMinutes: 3 + + server: + server: + httpPort: 8088 + grpcPort: 8089 + security: + secure: false + useAuth: false + allowCors: true + allowedOrigins: + # Accepting all domains for Sandbox installation + - "*" + allowedHeaders: + - "Content-Type" + flyteadmin: + roleNameKey: "iam.amazonaws.com/role" + profilerPort: 10254 + metricsScope: "flyte:" + metadataStoragePrefix: + - "metadata" + - "admin" + testing: + host: http://flyteadmin + scheduler: + eventScheduler: + scheme: aws + region: us-east-1 + scheduleRole: arn:aws:iam:::role/flyte_cron_scheduler_role + targetName: arn:aws:sqs:us-east-1::flyte-cron-scheduler-queue + scheduleNamePrefix: flyte + workflowExecutor: + scheme: aws + region: us-east-1 + scheduleQueueName: flyte-cron-scheduler-queue + accountId: + reconnectAttempts: 10 + reconnectDelaySeconds: 30 + notifications: + type: aws + region: us-east-1 + publisher: + topicName: arn:aws:sns:us-east-1::flyte-notifications-topic + processor: + queueName: flyte-notifications-queue + accountId: + emailer: + subject: "Flyte: {{ project }}/{{ domain }}/{{ launch_plan.name }} has '{{ phase }}'" + sender: "flyte@example.com" + body: | + Execution {{ workflow.project }}/{{ workflow.domain }}/{{ workflow.name }}/{{ name }} has {{ phase }}. + Details: https://flyte.example.com/console/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. + {{ error }} + task_type_whitelist: + spark: + - project: flytetester + - project: spark-workflows + + + datacatalogServer: + datacatalog: + storage-prefix: metadata/datacatalog + metrics-scope: datacatalog + profiler-port: 10254 + + storage: + storage: + type: s3 + container: s3-bucket-for-flyte + connection: + auth-type: iam + region: us-east-1 + + task_resource_defaults: + task_resources: + defaults: + cpu: 100m + memory: 1000Mi + storage: 5Mi + limits: + cpu: 15000m + memory: 100Gi + storage: 400Gi + + admin: + event: + type: admin + rate: 500 + capacity: 1000 + admin: + endpoint: flyteadmin:81 + insecure: true + + catalog: + catalog-cache: + endpoint: datacatalog:89 + type: datacatalog + insecure: true + + catalog_cache: + plugins: + catalogCache: + reader: + maxItems: 10000 + writer: + maxItems: 10000 + + copilot: + plugins: + k8s: + co-pilot: + name: flyte-copilot- + image: docker.io/lyft/flytecopilot:v0.3.35 + start-timeout: 30s + + core: + propeller: + rawoutput-prefix: s3://s3-bucket-for-flyte/ + metadata-prefix: metadata/propeller + workers: 40 + gc-interval: 12h + max-workflow-retries: 50 + workflow-reeval-duration: 30s + downstream-eval-duration: 30s + limit-namespace: "all" + prof-port: 10254 + metrics-prefix: flyte + enable-admin-launcher: true + leader-election: + lock-config-map: + name: propeller-leader + namespace: flyte + enabled: true + lease-duration: 15s + renew-deadline: 10s + retry-period: 2s + kube-client-config: + qps: 100 + burst: 25 + timeout: 30s + queue: + type: batch + batching-interval: 2s + batch-size: -1 + queue: + type: maxof + rate: 100 + capacity: 1000 + base-delay: 5s + max-delay: 120s + sub-queue: + type: bucket + rate: 100 + capacity: 1000 + workflowStore: + policy: "ResourceVersionCache" + + enabled_plugins: + tasks: + max-plugin-phase-versions: 1000000 + task-plugins: + enabled-plugins: + - container + - sidecar + - spark + - k8s-array + - pytorch + + k8s: + plugins: + k8s: + default-cpus: 100m + default-memory: 100Mi + + logger: + logger: + show-source: true + level: 5 + + qubole: + plugins: + qubole: + quboleTokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN" + + resource_manager: + propeller: + resourcemanager: + type: redis + resourceMaxQuota: 10000 + redis: + hostPath: redis-resource-manager:6379 + hostKey: mypassword + + spark: + plugins: + spark: + spark-config-default: + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" + - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.blacklist.enabled: "true" + - spark.blacklist.timeout: "5m" + - spark.task.maxfailures: "8" + + task_logs: + plugins: + logs: + kubernetes-enabled: false + + aa_namespace: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes + + ab_project_resource_quota: | + apiVersion: v1 + kind: ResourceQuota + metadata: + name: project-quota + namespace: {{ namespace }} + spec: + hard: + limits.cpu: {{ projectQuotaCpu }} + limits.memory: {{ projectQuotaMemory }} + + ac_project_copilot_dataconfig: | + kind: ConfigMap + apiVersion: v1 + metadata: + name: flyte-data-config + namespace: {{ namespace }} + data: + config.yaml: | + storage: + connection: + auth-type: iam + region: us-east-1 + type: s3 + container: my-s3-bucket + enable-multicontainer: true + + ad_spark_role: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: Role + metadata: + name: spark-role + namespace: {{ namespace }} + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - '*' + - apiGroups: + - "" + resources: + - services + verbs: + - '*' + - apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' + + ae_spark_service_account: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: spark + namespace: {{ namespace }} + + af_spark_role_binding: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: RoleBinding + metadata: + name: spark-role-binding + namespace: {{ namespace }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spark-role + subjects: + - kind: ServiceAccount + name: spark + namespace: {{ namespace }} diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml new file mode 100644 index 0000000000..41a351c741 --- /dev/null +++ b/helm/values-sandbox.yaml @@ -0,0 +1,146 @@ + +# +# FLYTEADMIN +# + +flyteadmin: {} # use default config + +# +# DATACATALOG +# + +datacatalog: {} + +# +# FLYTEPROPELLER +# + +flytepropeller: {} + +# +# FLYTECONSOLE +# + +flyteconsole: {} + +# +# REDIS +# + +redis: {} + +# +# POSTGRES +# + +postgres: {} + +# +# MINIO +# + +minio: {} + +# +# CONTOUR +# + +contour: {} + +# +# SPARKOPERATOR +# + +sparkoperator: {} + +# +# PYTORCHOPERATOR +# + +pytorchoperator: {} + +# +# COMMON +# + +common: {} + +# +# CONFIGMAPS +# + +configmap: + + db: + database: + port: 5432 + username: postgres + host: postgres + dbname: flyte + + storage: + storage: + type: minio + container: my-s3-bucket + connection: + access-key: minio + auth-type: accesskey + secret-key: miniostorage + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + + enabled_plugins: + tasks: + max-plugin-phase-versions: 1000000 + task-plugins: + enabled-plugins: + - container + - sidecar + - spark + - k8s-array + - pytorch + + k8s: + plugins: + k8s: + default-env-vars: + - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" + - FLYTE_AWS_ACCESS_KEY_ID: minio + - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage + + logger: + logger: + show-source: true + level: 5 + + ab_project_resource_quota: | + apiVersion: v1 + kind: ResourceQuota + metadata: + name: project-quota + namespace: {{ namespace }} + spec: + hard: + limits.cpu: {{ projectQuotaCpu }} + limits.memory: {{ projectQuotaMemory }} + + ac_project_copilot_dataconfig: | + kind: ConfigMap + apiVersion: v1 + metadata: + name: flyte-data-config + namespace: {{ namespace }} + data: + config.yaml: | + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + type: minio + container: my-s3-bucket + enable-multicontainer: true diff --git a/helm/values.yaml b/helm/values.yaml new file mode 100644 index 0000000000..67497dafa6 --- /dev/null +++ b/helm/values.yaml @@ -0,0 +1,550 @@ + +# +# FLYTEADMIN +# + +flyteadmin: + replicaCount: 1 + image: + repository: docker.io/lyft/flyteadmin + tag: v0.3.7 + pullPolicy: IfNotPresent + serviceAccountAnnotations: {} + resources: + limits: + cpu: 250m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + configPath: /etc/flyte/config/*.yaml + service: + annotations: + contour.heptio.com/upstream-protocol.h2c: grpc + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# DATACATALOG +# + +datacatalog: + replicaCount: 1 + image: + repository: docker.io/lyft/datacatalog + tag: v0.2.2 + pullPolicy: IfNotPresent + serviceAccountAnnotations: {} + resources: + limits: + cpu: 500m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + configPath: /etc/datacatalog/config/*.yaml + service: + annotations: + contour.heptio.com/upstream-protocol.h2c: grpc + type: NodePort + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# FLYTEPROPELLER +# + +flytepropeller: + replicaCount: 1 + image: + repository: docker.io/lyft/flytepropeller + tag: v0.4.2 + pullPolicy: IfNotPresent + serviceAccountAnnotations: {} + resources: + limits: + cpu: 200m + ephemeral-storage: 100Mi + memory: 200Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + configPath: /etc/flyte/config/*.yaml + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# FLYTECONSOLE +# + +flyteconsole: + replicaCount: 1 + image: + repository: docker.io/lyft/flyteconsole + tag: v0.12.1 + pullPolicy: IfNotPresent + resources: + limits: + cpu: 500m + memory: 250Mi + requests: + cpu: 10m + memory: 50Mi + service: {} + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# REDIS +# + +redis: + enabled: true # Set false to define malternative redis + replicaCount: 1 + image: + repository: docker.io/bitnami/redis + tag: 4.0.2-r1 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 10m + memory: 50Mi + service: {} + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# POSTGRES +# + +postgres: + enabled: true # Set false if you are going to use RDS + replicaCount: 1 + image: + repository: postgres + tag: "10.1" + pullPolicy: IfNotPresent + resources: + requests: + cpu: 10m + memory: 128Mi + service: {} + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# MINIO +# + +minio: + enabled: true # Set false if you'd like using S3 + replicaCount: 1 + image: + repository: minio/minio + tag: RELEASE.2019-06-04T01-15-58Z + pullPolicy: IfNotPresent + resources: + requests: + cpu: 10m + memory: 128Mi + service: {} + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# CONTOUR +# + +contour: + enabled: true # Set false if you have alternative ingress controller like Istio or Ingress-Nginx + replicaCount: 1 + contour: + image: + repository: gcr.io/heptio-images/contour + tag: v0.6.1 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + cpu: 100m + memory: 100Mi + envoy: + image: + repository: docker.io/envoyproxy/envoy-alpine + tag: v1.6.0 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + cpu: 100m + memory: 100Mi + service: + annotations: + type: NodePort + ports: + - protocol: TCP + port: 80 + nodePort: 30081 + serviceAccountAnnotations: {} + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# SPARKOPERATOR +# + +sparkoperator: + enabled: true # Set false to disable + replicaCount: 1 + image: + repository: gcr.io/spark-operator/spark-operator + tag: v2.4.0-v1beta1-0.9.0 + pullPolicy: IfNotPresent + serviceAccountAnnotations: {} + securityContext: {} + service: {} + resources: + limits: + cpu: 1000m + memory: 500M + requests: + cpu: 10m + memory: 50M + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# PYTORCHOPERATOR +# + +pytorchoperator: + enabled: true # Set false to disable + replicaCount: 1 + image: + repository: gcr.io/kubeflow-images-public/pytorch-operator + tag: v1.0.0-g047cf0f + pullPolicy: IfNotPresent + serviceAccountAnnotations: {} + service: {} + resources: + limits: + cpu: 500m + memory: 1000M + requests: + cpu: 10m + memory: 50M + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# +# COMMON +# + +common: + databaseSecret: + create: false + name: + ingressAnnotations: {} + # nginx.ingress.kubernetes.io/ssl-redirect: "false" + +# +# CONFIGMAPS +# + +configmap: + + console: + BASE_URL: /console + CONFIG_DIR: /etc/flyte/config + + cluster_resources: + cluster_resources: + templatePath: "/etc/flyte/clusterresource/templates" + customData: + production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "3000Mi" + refresh: 5m + + db: + database: {} + + domain: + domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + + remote_data: + remoteData: + region: us-east-1 + scheme: local + signedUrls: + durationMinutes: 3 + + server: + server: + httpPort: 8088 + grpcPort: 8089 + security: + secure: false + useAuth: false + allowCors: true + allowedOrigins: + # Accepting all domains for Sandbox installation + - "*" + allowedHeaders: + - "Content-Type" + flyteadmin: + roleNameKey: "iam.amazonaws.com/role" + profilerPort: 10254 + metricsScope: "flyte:" + metadataStoragePrefix: + - "metadata" + - "admin" + testing: + host: http://flyteadmin + + datacatalogServer: + datacatalog: + storage-prefix: metadata/datacatalog + metrics-scope: datacatalog + profiler-port: 10254 + + storage: + storage: + type: minio + container: my-s3-bucket + connection: + auth-type: accesskey + region: us-east-1 + + task_resource_defaults: + task_resources: + defaults: + cpu: 100m + memory: 1000Mi + storage: 5Mi + limits: + cpu: 15000m + memory: 100Gi + storage: 400Gi + + admin: + event: + type: admin + rate: 500 + capacity: 1000 + admin: + endpoint: flyteadmin:81 + insecure: true + + catalog: + catalog-cache: + endpoint: datacatalog:89 + type: datacatalog + insecure: true + + catalog_cache: + plugins: + catalogCache: + reader: + maxItems: 10000 + writer: + maxItems: 10000 + + copilot: + plugins: + k8s: + co-pilot: + name: flyte-copilot- + image: docker.io/lyft/flytecopilot:v0.3.35 + start-timeout: 30s + + core: + propeller: + rawoutput-prefix: s3://my-s3-bucket/ + metadata-prefix: metadata/propeller + workers: 4 + max-workflow-retries: 30 + workflow-reeval-duration: 30s + downstream-eval-duration: 30s + limit-namespace: "all" + prof-port: 10254 + metrics-prefix: flyte + enable-admin-launcher: true + leader-election: + lock-config-map: + name: propeller-leader + namespace: flyte + enabled: true + lease-duration: 15s + renew-deadline: 10s + retry-period: 2s + queue: + type: batch + batching-interval: 2s + batch-size: -1 + queue: + type: bucket + rate: 10 + capacity: 100 + sub-queue: + type: bucket + rate: 10 + capacity: 100 + + enabled_plugins: + tasks: + max-plugin-phase-versions: 1000000 + task-plugins: + enabled-plugins: [] + + k8s: + plugins: + k8s: + default-cpus: 100m + default-memory: 100Mi + + logger: + logger: + show-source: true + level: 4 + + qubole: + plugins: + qubole: + quboleTokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN" + + resource_manager: + propeller: + resourcemanager: + type: redis + resourceMaxQuota: 10000 + redis: + hostPath: redis-resource-manager:6379 + hostKey: mypassword + + spark: + plugins: + spark: + spark-config-default: + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" + - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.blacklist.enabled: "true" + - spark.blacklist.timeout: "5m" + - spark.task.maxfailures: "8" + + task_logs: + plugins: + logs: + kubernetes-enabled: true + + aa_namespace: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes + + ad_spark_role: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: Role + metadata: + name: spark-role + namespace: {{ namespace }} + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - '*' + - apiGroups: + - "" + resources: + - services + verbs: + - '*' + - apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' + + ae_spark_service_account: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: spark + namespace: {{ namespace }} + + af_spark_role_binding: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: RoleBinding + metadata: + name: spark-role-binding + namespace: {{ namespace }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spark-role + subjects: + - kind: ServiceAccount + name: spark + namespace: {{ namespace }} From 4908da10ac32f287c69dee91ee3428b6df76de94 Mon Sep 17 00:00:00 2001 From: Ruslan Stanevich Date: Tue, 13 Oct 2020 22:06:54 +0300 Subject: [PATCH 03/92] remove new lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/_helpers.tpl | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 189f8f89e8..398d189543 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -1,7 +1,5 @@ {{/* vim: set filetype=mustache: */}} - - {{- define "flyte.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -15,7 +13,6 @@ {{- end -}} - {{- define "flyteadmin.name" -}} flyteadmin {{- end -}} @@ -32,7 +29,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - {{- define "datacatalog.name" -}} datacatalog {{- end -}} @@ -49,7 +45,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - {{- define "flytepropeller.name" -}} flytepropeller {{- end -}} @@ -66,7 +61,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - {{- define "flyteconsole.name" -}} flyteconsole {{- end -}} @@ -83,7 +77,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - {{- define "redis.name" -}} redis {{- end -}} @@ -100,7 +93,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - {{- define "postgres.name" -}} postgres {{- end -}} @@ -117,7 +109,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - {{- define "minio.name" -}} minio {{- end -}} @@ -154,7 +145,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - {{- define "sparkoperator.name" -}} sparkoperator {{- end -}} @@ -175,7 +165,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - {{- define "pytorch-operator.name" -}} pytorch-operator {{- end -}} @@ -195,10 +184,7 @@ helm.sh/chart: {{ include "flyte.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - -# -# SECRET -# +# Optional blocks for secret mount {{- define "databaseSecret.volume" -}} {{- with .Values.common.databaseSecret.name -}} From 90a7422e479d8e8d273733849807fa43845ea43e Mon Sep 17 00:00:00 2001 From: Ruslan Stanevich Date: Wed, 14 Oct 2020 18:12:18 +0300 Subject: [PATCH 04/92] Added autogenerated Helm docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/README.MD | 236 ++++++++++++++++++++++ helm/README.md.gotmpl | 28 +++ helm/templates/admin/configmap.yaml | 39 ++-- helm/templates/common/configmap.yaml | 2 + helm/templates/common/ingress.yaml | 4 +- helm/templates/common/namespace.yaml | 2 + helm/templates/common/secret.yaml | 1 - helm/templates/datacatalog/configmap.yaml | 19 +- helm/templates/propeller/configmap.yaml | 64 +++--- helm/values-eks.yaml | 6 +- helm/values-sandbox.yaml | 13 +- helm/values.yaml | 230 +++++++++++++++++---- 12 files changed, 554 insertions(+), 90 deletions(-) create mode 100644 helm/README.md.gotmpl diff --git a/helm/README.MD b/helm/README.MD index bd8379c8b7..26a6dc142b 100644 --- a/helm/README.MD +++ b/helm/README.MD @@ -1,3 +1,9 @@ +# flyte + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.8.0](https://img.shields.io/badge/AppVersion-0.8.0-informational?style=flat-square) + +A Helm chart for Flyte + ### SANDBOX INSTALLATION: - Install `helm 3` realese version - Generate `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` @@ -9,3 +15,233 @@ - The docker images, their tags and other default parameters are configured in `values.yaml` file. - Each Flyte installation type should have separate `values-*.yaml` file: for sandbox, EKS and etc. The configuration in `values.yaml` and the choosen config `values-*.yaml` are merged when generating the deployment manifest. - The configuration in `values-sandbox.yaml` is ready for installation in minikube. But `values-eks.yaml` should be edited before installation: s3 bucket, RDS hosts, iam roles, secrets and etc need to be modified. + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| common.databaseSecret.name | string | `""` | Specify name of K8s Secret which contains Database password. Leave it empty if you don't need this Secret | +| common.databaseSecret.secretManifest | object | `{}` | Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets | +| common.flyteNamespaceTemplate.enabled | bool | `true` | | +| common.ingress.annotations | object | `{}` | | +| common.ingress.enabled | bool | `true` | | +| configmap.aa_namespace | string | `"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"` | Template for namespaces resources | +| configmap.ad_spark_role | string | `"apiVersion: rbac.authorization.k8s.io/v1beta1\nkind: Role\nmetadata:\n name: spark-role\n namespace: {{ namespace }}\nrules:\n- apiGroups:\n - \"\"\n resources:\n - pods\n verbs:\n - '*'\n- apiGroups:\n - \"\"\n resources:\n - services\n verbs:\n - '*'\n- apiGroups:\n - \"\"\n resources:\n - configmaps\n verbs:\n - '*'\n"` | Template for spark role | +| configmap.admin.admin.endpoint | string | `"flyteadmin:81"` | | +| configmap.admin.admin.insecure | bool | `true` | | +| configmap.admin.event.capacity | int | `1000` | | +| configmap.admin.event.rate | int | `500` | | +| configmap.admin.event.type | string | `"admin"` | | +| configmap.ae_spark_service_account | string | `"apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: spark\n namespace: {{ namespace }}\n"` | Template for spark SA | +| configmap.af_spark_role_binding | string | `"apiVersion: rbac.authorization.k8s.io/v1beta1\nkind: RoleBinding\nmetadata:\n name: spark-role-binding\n namespace: {{ namespace }} \nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: spark-role\nsubjects:\n- kind: ServiceAccount\n name: spark\n namespace: {{ namespace }}\n"` | Template for spark role binding | +| configmap.catalog.catalog-cache.endpoint | string | `"datacatalog:89"` | | +| configmap.catalog.catalog-cache.insecure | bool | `true` | | +| configmap.catalog.catalog-cache.type | string | `"datacatalog"` | | +| configmap.catalog_cache.plugins.catalogCache.reader.maxItems | int | `10000` | | +| configmap.catalog_cache.plugins.catalogCache.writer.maxItems | int | `10000` | | +| configmap.cluster_resources.cluster_resources.customData.development[0].projectQuotaCpu.value | string | `"4"` | | +| configmap.cluster_resources.cluster_resources.customData.development[1].projectQuotaMemory.value | string | `"3000Mi"` | | +| configmap.cluster_resources.cluster_resources.customData.production[0].projectQuotaCpu.value | string | `"5"` | | +| configmap.cluster_resources.cluster_resources.customData.production[1].projectQuotaMemory.value | string | `"4000Mi"` | | +| configmap.cluster_resources.cluster_resources.customData.staging[0].projectQuotaCpu.value | string | `"2"` | | +| configmap.cluster_resources.cluster_resources.customData.staging[1].projectQuotaMemory.value | string | `"3000Mi"` | | +| configmap.cluster_resources.cluster_resources.refresh | string | `"5m"` | | +| configmap.cluster_resources.cluster_resources.templatePath | string | `"/etc/flyte/clusterresource/templates"` | | +| configmap.console.BASE_URL | string | `"/console"` | | +| configmap.console.CONFIG_DIR | string | `"/etc/flyte/config"` | | +| configmap.copilot.plugins.k8s.co-pilot.image | string | `"docker.io/lyft/flytecopilot:v0.3.35"` | | +| configmap.copilot.plugins.k8s.co-pilot.name | string | `"flyte-copilot-"` | | +| configmap.copilot.plugins.k8s.co-pilot.start-timeout | string | `"30s"` | | +| configmap.core.propeller.downstream-eval-duration | string | `"30s"` | | +| configmap.core.propeller.enable-admin-launcher | bool | `true` | | +| configmap.core.propeller.leader-election.enabled | bool | `true` | | +| configmap.core.propeller.leader-election.lease-duration | string | `"15s"` | | +| configmap.core.propeller.leader-election.lock-config-map.name | string | `"propeller-leader"` | | +| configmap.core.propeller.leader-election.lock-config-map.namespace | string | `"flyte"` | | +| configmap.core.propeller.leader-election.renew-deadline | string | `"10s"` | | +| configmap.core.propeller.leader-election.retry-period | string | `"2s"` | | +| configmap.core.propeller.limit-namespace | string | `"all"` | | +| configmap.core.propeller.max-workflow-retries | int | `30` | | +| configmap.core.propeller.metadata-prefix | string | `"metadata/propeller"` | | +| configmap.core.propeller.metrics-prefix | string | `"flyte"` | | +| configmap.core.propeller.prof-port | int | `10254` | | +| configmap.core.propeller.queue.batch-size | int | `-1` | | +| configmap.core.propeller.queue.batching-interval | string | `"2s"` | | +| configmap.core.propeller.queue.queue.capacity | int | `100` | | +| configmap.core.propeller.queue.queue.rate | int | `10` | | +| configmap.core.propeller.queue.queue.type | string | `"bucket"` | | +| configmap.core.propeller.queue.sub-queue.capacity | int | `100` | | +| configmap.core.propeller.queue.sub-queue.rate | int | `10` | | +| configmap.core.propeller.queue.sub-queue.type | string | `"bucket"` | | +| configmap.core.propeller.queue.type | string | `"batch"` | | +| configmap.core.propeller.rawoutput-prefix | string | `"s3://my-s3-bucket/"` | | +| configmap.core.propeller.workers | int | `4` | | +| configmap.core.propeller.workflow-reeval-duration | string | `"30s"` | | +| configmap.datacatalogServer.datacatalog.metrics-scope | string | `"datacatalog"` | | +| configmap.datacatalogServer.datacatalog.profiler-port | int | `10254` | | +| configmap.datacatalogServer.datacatalog.storage-prefix | string | `"metadata/datacatalog"` | | +| configmap.db.database | object | `{}` | | +| configmap.domain.domains[0].id | string | `"development"` | | +| configmap.domain.domains[0].name | string | `"development"` | | +| configmap.domain.domains[1].id | string | `"staging"` | | +| configmap.domain.domains[1].name | string | `"staging"` | | +| configmap.domain.domains[2].id | string | `"production"` | | +| configmap.domain.domains[2].name | string | `"production"` | | +| configmap.enabled_plugins.tasks.max-plugin-phase-versions | int | `1000000` | | +| configmap.enabled_plugins.tasks.task-plugins.enabled-plugins | list | `[]` | | +| configmap.k8s.plugins.k8s.default-cpus | string | `"100m"` | | +| configmap.k8s.plugins.k8s.default-env-vars | list | `[]` | | +| configmap.k8s.plugins.k8s.default-memory | string | `"100Mi"` | | +| configmap.logger.logger.level | int | `4` | | +| configmap.logger.logger.show-source | bool | `true` | | +| configmap.qubole.plugins.qubole.quboleTokenKey | string | `"FLYTE_QUBOLE_CLIENT_TOKEN"` | | +| configmap.remote_data.remoteData.region | string | `"us-east-1"` | | +| configmap.remote_data.remoteData.scheme | string | `"local"` | | +| configmap.remote_data.remoteData.signedUrls.durationMinutes | int | `3` | | +| configmap.resource_manager.propeller.resourcemanager.redis.hostKey | string | `"mypassword"` | | +| configmap.resource_manager.propeller.resourcemanager.redis.hostPath | string | `"redis-resource-manager:6379"` | | +| configmap.resource_manager.propeller.resourcemanager.resourceMaxQuota | int | `10000` | | +| configmap.resource_manager.propeller.resourcemanager.type | string | `"redis"` | | +| configmap.server.flyteadmin.metadataStoragePrefix[0] | string | `"metadata"` | | +| configmap.server.flyteadmin.metadataStoragePrefix[1] | string | `"admin"` | | +| configmap.server.flyteadmin.metricsScope | string | `"flyte:"` | | +| configmap.server.flyteadmin.profilerPort | int | `10254` | | +| configmap.server.flyteadmin.roleNameKey | string | `"iam.amazonaws.com/role"` | | +| configmap.server.flyteadmin.testing.host | string | `"http://flyteadmin"` | | +| configmap.server.server.grpcPort | int | `8089` | | +| configmap.server.server.httpPort | int | `8088` | | +| configmap.server.server.security.allowCors | bool | `true` | | +| configmap.server.server.security.allowedHeaders[0] | string | `"Content-Type"` | | +| configmap.server.server.security.allowedOrigins[0] | string | `"*"` | | +| configmap.server.server.security.secure | bool | `false` | | +| configmap.server.server.security.useAuth | bool | `false` | | +| configmap.spark.plugins.spark.spark-config-default | list | `[{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]` | Spark default configuration | +| configmap.storage.storage.connection.auth-type | string | `"accesskey"` | | +| configmap.storage.storage.connection.region | string | `"us-east-1"` | | +| configmap.storage.storage.container | string | `"my-s3-bucket"` | | +| configmap.storage.storage.type | string | `"minio"` | | +| configmap.task_logs.plugins.logs.kubernetes-enabled | bool | `false` | | +| configmap.task_resource_defaults.task_resources.defaults.cpu | string | `"100m"` | | +| configmap.task_resource_defaults.task_resources.defaults.memory | string | `"1000Mi"` | | +| configmap.task_resource_defaults.task_resources.defaults.storage | string | `"5Mi"` | | +| configmap.task_resource_defaults.task_resources.limits.cpu | string | `"15000m"` | | +| configmap.task_resource_defaults.task_resources.limits.memory | string | `"100Gi"` | | +| configmap.task_resource_defaults.task_resources.limits.storage | string | `"400Gi"` | | +| contour.affinity | object | `{}` | affinity for Contour deployment | +| contour.contour.image.pullPolicy | string | `"IfNotPresent"` | | +| contour.contour.image.repository | string | `"gcr.io/heptio-images/contour"` | Docker image for Contour | +| contour.contour.image.tag | string | `"v0.6.1"` | | +| contour.contour.resources | object | `{"limits":{"cpu":"100m","memory":"100Mi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Contour | +| contour.enabled | bool | `true` | | +| contour.envoy.image.pullPolicy | string | `"IfNotPresent"` | | +| contour.envoy.image.repository | string | `"docker.io/envoyproxy/envoy-alpine"` | Docker image for Envoy | +| contour.envoy.image.tag | string | `"v1.6.0"` | | +| contour.envoy.resources | object | `{"limits":{"cpu":"100m","memory":"100Mi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Envoy | +| contour.nodeSelector | object | `{}` | nodeSelector for Contour deployment | +| contour.podAnnotations | object | `{}` | Annotations for Contour pods | +| contour.replicaCount | int | `1` | Replicas count for Contour deployment | +| contour.service | object | `{"annotations":{},"ports":[{"nodePort":30081,"port":80,"protocol":"TCP"}],"type":"NodePort"}` | Service settings for Contour | +| contour.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Contour pods | +| contour.tolerations | list | `[]` | tolerations for Contour deployment | +| datacatalog.affinity | object | `{}` | affinity for Datacatalog deployment | +| datacatalog.configPath | string | `"/etc/datacatalog/config/*.yaml"` | Default regex string for searching configuration files | +| datacatalog.image.pullPolicy | string | `"IfNotPresent"` | | +| datacatalog.image.repository | string | `"docker.io/lyft/datacatalog"` | Docker image for Datacatalog deployment | +| datacatalog.image.tag | string | `"v0.2.2"` | | +| datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | +| datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | +| datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | +| datacatalog.resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Datacatalog deployment | +| datacatalog.service | object | `{"annotations":{"contour.heptio.com/upstream-protocol.h2c":"grpc"},"type":"NodePort"}` | Service settings for Datacatalog | +| datacatalog.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Datacatalog pods | +| datacatalog.tolerations | list | `[]` | tolerations for Datacatalog deployment | +| flyteadmin.affinity | object | `{}` | affinity for Flyteadmin deployment | +| flyteadmin.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | +| flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | | +| flyteadmin.image.repository | string | `"docker.io/lyft/flyteadmin"` | Docker image for Flyteadmin deployment | +| flyteadmin.image.tag | string | `"v0.3.7"` | | +| flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | +| flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | +| flyteadmin.replicaCount | int | `1` | Replicas count for Flyteadmin deployment | +| flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment | +| flyteadmin.service | object | `{"annotations":{"contour.heptio.com/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"}` | Service settings for Flyteadmin | +| flyteadmin.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Flyteadmin pods | +| flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment | +| flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | +| flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | | +| flyteconsole.image.repository | string | `"docker.io/lyft/flyteconsole"` | Docker image for Flyteconsole deployment | +| flyteconsole.image.tag | string | `"v0.12.1"` | | +| flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | +| flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | +| flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment | +| flyteconsole.resources | object | `{"limits":{"cpu":"500m","memory":"250Mi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Flyteconsole deployment | +| flyteconsole.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Flyteconsole | +| flyteconsole.tolerations | list | `[]` | tolerations for Flyteconsole deployment | +| flytepropeller.affinity | object | `{}` | affinity for Flytepropeller deployment | +| flytepropeller.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | +| flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | | +| flytepropeller.image.repository | string | `"docker.io/lyft/flytepropeller"` | Docker image for Flytepropeller deployment | +| flytepropeller.image.tag | string | `"v0.4.2"` | | +| flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | +| flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | +| flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | +| flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytepropeller deployment | +| flytepropeller.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Flytepropeller pods | +| flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment | +| minio.affinity | object | `{}` | affinity for Minio deployment | +| minio.enabled | bool | `true` | | +| minio.image.pullPolicy | string | `"IfNotPresent"` | | +| minio.image.repository | string | `"minio/minio"` | Docker image for Minio deployment | +| minio.image.tag | string | `"RELEASE.2019-06-04T01-15-58Z"` | | +| minio.nodeSelector | object | `{}` | nodeSelector for Minio deployment | +| minio.podAnnotations | object | `{}` | Annotations for Minio pods | +| minio.replicaCount | int | `1` | Replicas count for Minio deployment | +| minio.resources | object | `{"limits":{"cpu":"200m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}}` | Default resources requests and limits for Minio deployment | +| minio.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Minio | +| minio.tolerations | list | `[]` | tolerations for Minio deployment | +| postgres.affinity | object | `{}` | affinity for Postgres deployment | +| postgres.enabled | bool | `true` | | +| postgres.image.pullPolicy | string | `"IfNotPresent"` | | +| postgres.image.repository | string | `"postgres"` | Docker image for Postgres deployment | +| postgres.image.tag | string | `"10.1"` | | +| postgres.nodeSelector | object | `{}` | nodeSelector for Postgres deployment | +| postgres.podAnnotations | object | `{}` | Annotations for Postgres pods | +| postgres.replicaCount | int | `1` | Replicas count for Postgres deployment | +| postgres.resources | object | `{"limits":{"cpu":"1000m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}}` | Default resources requests and limits for Postgres deployment | +| postgres.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Postgres | +| postgres.tolerations | list | `[]` | tolerations for Postgres deployment | +| pytorchoperator.affinity | object | `{}` | affinity for Pytorchoperator deployment | +| pytorchoperator.enabled | bool | `true` | | +| pytorchoperator.image.pullPolicy | string | `"IfNotPresent"` | | +| pytorchoperator.image.repository | string | `"gcr.io/kubeflow-images-public/pytorch-operator"` | Docker image for Pytorchoperator | +| pytorchoperator.image.tag | string | `"v1.0.0-g047cf0f"` | | +| pytorchoperator.nodeSelector | object | `{}` | nodeSelector for Pytorchoperator deployment | +| pytorchoperator.podAnnotations | object | `{}` | Annotations for Pytorchoperator pods | +| pytorchoperator.replicaCount | int | `1` | Replicas count for Pytorchoperator deployment | +| pytorchoperator.resources | object | `{"limits":{"cpu":"500m","memory":"1000M"},"requests":{"cpu":"10m","memory":"50M"}}` | Default resources requests and limits for Pytorchoperator | +| pytorchoperator.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Pytorchoperator | +| pytorchoperator.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Pytorchoperator pods | +| pytorchoperator.tolerations | list | `[]` | tolerations for Pytorchoperator deployment | +| redis.affinity | object | `{}` | affinity for Redis Statefulset | +| redis.enabled | bool | `true` | | +| redis.image.pullPolicy | string | `"IfNotPresent"` | | +| redis.image.repository | string | `"docker.io/bitnami/redis"` | Docker image for Redis Statefulset | +| redis.image.tag | string | `"4.0.2-r1"` | | +| redis.nodeSelector | object | `{}` | nodeSelector for Redis Statefulset | +| redis.podAnnotations | object | `{}` | Annotations for Redis pods | +| redis.replicaCount | int | `1` | Replicas count for Redis Statefulset | +| redis.resources | object | `{"limits":{"cpu":"1000m","memory":"1Gi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Redis Statefulset | +| redis.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Redis | +| redis.tolerations | list | `[]` | tolerations for Redis Statefulset | +| sparkoperator.affinity | object | `{}` | affinity for Sparkoperator deployment | +| sparkoperator.enabled | bool | `true` | | +| sparkoperator.image.pullPolicy | string | `"IfNotPresent"` | | +| sparkoperator.image.repository | string | `"gcr.io/spark-operator/spark-operator"` | Docker image for Sparkoperator | +| sparkoperator.image.tag | string | `"v2.4.0-v1beta1-0.9.0"` | | +| sparkoperator.nodeSelector | object | `{}` | nodeSelector for Sparkoperator deployment | +| sparkoperator.podAnnotations | object | `{}` | Annotations for Sparkoperator pods | +| sparkoperator.replicaCount | int | `1` | Replicas count for Sparkoperator deployment | +| sparkoperator.resources | object | `{"limits":{"cpu":"1000m","memory":"500M"},"requests":{"cpu":"10m","memory":"50M"}}` | Default resources requests and limits for Sparkoperator | +| sparkoperator.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Contour | +| sparkoperator.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Sparkoperator pods | +| sparkoperator.tolerations | list | `[]` | tolerations for Sparkoperator deployment | diff --git a/helm/README.md.gotmpl b/helm/README.md.gotmpl new file mode 100644 index 0000000000..eb4f8c63d5 --- /dev/null +++ b/helm/README.md.gotmpl @@ -0,0 +1,28 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +### SANDBOX INSTALLATION: +- Install `helm 3` realese version +- Generate `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` +- Deploy the manifest `kubectl apply -f flyte_generated_sandbox.yaml` +- When all pods are running - run end2end tests: `kubectl apply -f ../end2end/tests/endtoend.yaml` +- Get flyte host `minikube service contour -n heptio-contour --url`. And then visit `http:///console` + +### CONFIGURATION NOTES: +- The docker images, their tags and other default parameters are configured in `values.yaml` file. +- Each Flyte installation type should have separate `values-*.yaml` file: for sandbox, EKS and etc. The configuration in `values.yaml` and the choosen config `values-*.yaml` are merged when generating the deployment manifest. +- The configuration in `values-sandbox.yaml` is ready for installation in minikube. But `values-eks.yaml` should be edited before installation: s3 bucket, RDS hosts, iam roles, secrets and etc need to be modified. + +{{ template "chart.valuesSection" . }} diff --git a/helm/templates/admin/configmap.yaml b/helm/templates/admin/configmap.yaml index 75921682d2..a061058caa 100644 --- a/helm/templates/admin/configmap.yaml +++ b/helm/templates/admin/configmap.yaml @@ -5,18 +5,27 @@ metadata: namespace: {{ template "flyte.namespace" . }} labels: {{ include "flyteadmin.labels" . | nindent 4 }} data: - cluster_resources.yaml: | {{ toYaml .Values.configmap.cluster_resources | nindent 4 }} - - db.yaml: | {{ toYaml .Values.configmap.db | nindent 4 }} - - domain.yaml: | {{ toYaml .Values.configmap.domain | nindent 4 }} - - logger.yaml: | {{ toYaml .Values.configmap.logger | nindent 4 }} - - remote_data.yaml: | {{ toYaml .Values.configmap.remote_data | nindent 4 }} - - server.yaml: | {{ toYaml .Values.configmap.server | nindent 4 }} - - storage.yaml: | {{ toYaml .Values.configmap.storage | nindent 4 }} - - task_resource_defaults.yaml: | {{ toYaml .Values.configmap.task_resource_defaults | nindent 4 }} +{{- with .Values.configmap.cluster_resources }} + cluster_resources.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.db }} + db.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.domain }} + domain.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.logger }} + logger.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.remote_data }} + remote_data.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.server }} + server.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.storage }} + storage.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.task_resource_defaults }} + task_resource_defaults.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} diff --git a/helm/templates/common/configmap.yaml b/helm/templates/common/configmap.yaml index 606585656f..4291ec676f 100644 --- a/helm/templates/common/configmap.yaml +++ b/helm/templates/common/configmap.yaml @@ -14,6 +14,7 @@ data: {{- with .Values.configmap.ac_project_copilot_dataconfig }} ac_project-copilot-dataconfig.yaml: | {{ . | nindent 4 }} {{- end }} +{{- if .Values.sparkoperator.enabled }} {{- with .Values.configmap.ad_spark_role }} ad_spark-role.yaml: | {{ . | nindent 4 }} {{- end }} @@ -23,3 +24,4 @@ data: {{- with .Values.configmap.af_spark_role_binding }} af_spark-role-binding.yaml: | {{ . | nindent 4 }} {{- end }} +{{- end }} diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index 93c65bcbcf..67539b0658 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -1,9 +1,10 @@ +{{- if .Values.common.ingress.enabled }} apiVersion: extensions/v1beta1 kind: Ingress metadata: name: {{ template "flyte.name" . }} namespace: {{ template "flyte.namespace" . }} - {{- with .Values.common.ingressAnnotations }} + {{- with .Values.common.ingress.annotations }} annotations: {{ toYaml . | nindent 4}} {{- end }} spec: @@ -38,3 +39,4 @@ spec: serviceName: flyteadmin servicePort: 87 path: /openapi +{{- end }} \ No newline at end of file diff --git a/helm/templates/common/namespace.yaml b/helm/templates/common/namespace.yaml index bf158e57c9..c8a3dc17a7 100644 --- a/helm/templates/common/namespace.yaml +++ b/helm/templates/common/namespace.yaml @@ -1,4 +1,6 @@ +{{- if .Values.common.flyteNamespaceTemplate.enabled }} apiVersion: v1 kind: Namespace metadata: name: {{ template "flyte.namespace" . }} +{{- end }} diff --git a/helm/templates/common/secret.yaml b/helm/templates/common/secret.yaml index 7eb925de80..0282eda01d 100644 --- a/helm/templates/common/secret.yaml +++ b/helm/templates/common/secret.yaml @@ -1,4 +1,3 @@ {{- with .Values.common.databaseSecret.secretManifest }} {{ toYaml . }} {{- end }} - \ No newline at end of file diff --git a/helm/templates/datacatalog/configmap.yaml b/helm/templates/datacatalog/configmap.yaml index a552b02dac..dce73e47e4 100644 --- a/helm/templates/datacatalog/configmap.yaml +++ b/helm/templates/datacatalog/configmap.yaml @@ -5,10 +5,15 @@ metadata: namespace: {{ template "flyte.namespace" . }} labels: {{ include "datacatalog.labels" . | nindent 4 }} data: - db.yaml: | {{ toYaml .Values.configmap.db | nindent 4 }} - - logger.yaml: | {{ toYaml .Values.configmap.logger | nindent 4 }} - - server.yaml: | {{ toYaml .Values.configmap.datacatalogServer | nindent 4 }} - - storage.yaml: | {{ toYaml .Values.configmap.storage | nindent 4 }} +{{- with .Values.configmap.db }} + db.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.logger }} + logger.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.datacatalogServer }} + server.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.storage }} + storage.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} diff --git a/helm/templates/propeller/configmap.yaml b/helm/templates/propeller/configmap.yaml index 5d341c05bd..c604b5656e 100644 --- a/helm/templates/propeller/configmap.yaml +++ b/helm/templates/propeller/configmap.yaml @@ -5,28 +5,42 @@ metadata: namespace: {{ template "flyte.namespace" . }} labels: {{ include "flyteadmin.labels" . | nindent 4 }} data: - admin.yaml: | {{ toYaml .Values.configmap.admin | nindent 4 }} - - catalog.yaml: | {{ toYaml .Values.configmap.catalog | nindent 4 }} - - catalog_cache.yaml: | {{ toYaml .Values.configmap.catalog_cache | nindent 4 }} - - copilot.yaml: | {{ toYaml .Values.configmap.copilot | nindent 4 }} - - core.yaml: | {{ toYaml .Values.configmap.core | nindent 4 }} - - enabled_plugins.yaml: | {{ toYaml .Values.configmap.enabled_plugins | nindent 4 }} - - k8s.yaml: | {{ toYaml .Values.configmap.k8s | nindent 4 }} - - logger.yaml: | {{ toYaml .Values.configmap.logger | nindent 4 }} - - qubole.yaml: | {{ toYaml .Values.configmap.qubole | nindent 4 }} - - resource_manager.yaml: | {{ toYaml .Values.configmap.resource_manager | nindent 4 }} - - spark.yaml: | {{ toYaml .Values.configmap.spark | nindent 4 }} - - storage.yaml: | {{ toYaml .Values.configmap.storage | nindent 4 }} - - task_logs.yaml: | {{ toYaml .Values.configmap.task_logs | nindent 4 }} +{{- with .Values.configmap.admin }} + admin.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.catalog }} + catalog.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.catalog_cache }} + catalog_cache.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.copilot }} + copilot.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.core }} + core.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.enabled_plugins }} + enabled_plugins.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.k8s }} + k8s.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.logger }} + logger.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.qubole }} + qubole.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.resource_manager }} + resource_manager.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.spark }} + spark.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.storage }} + storage.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.task_logs }} + task_logs.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index d1a113a841..0059572e80 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -67,7 +67,7 @@ datacatalog: service: annotations: contour.heptio.com/upstream-protocol.h2c: grpc - + affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -155,14 +155,14 @@ redis: # postgres: - enabled: false # recommended adding managed RDS host to the config + enabled: false # # MINIO # minio: - enabled: false # Use AWS S3 bucket instead + enabled: false # # CONTOUR diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index 41a351c741..e5926b4b11 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -63,7 +63,10 @@ pytorchoperator: {} # COMMON # -common: {} +common: + databaseSecret: {} + ingress: {} + flyteNamespaceTemplate: {} # # CONFIGMAPS @@ -76,7 +79,7 @@ configmap: port: 5432 username: postgres host: postgres - dbname: flyte + dbname: flyte_development storage: storage: @@ -109,6 +112,12 @@ configmap: - FLYTE_AWS_ACCESS_KEY_ID: minio - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage + task_logs: + plugins: + logs: + kubernetes-enabled: true + kubernetes-url: http://localhost:30082 + logger: logger: show-source: true diff --git a/helm/values.yaml b/helm/values.yaml index 67497dafa6..5c6d1ca33a 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,15 +1,17 @@ # -# FLYTEADMIN +# FLYTEADMIN SETTINGS # flyteadmin: + # -- Replicas count for Flyteadmin deployment replicaCount: 1 image: + # -- Docker image for Flyteadmin deployment repository: docker.io/lyft/flyteadmin tag: v0.3.7 pullPolicy: IfNotPresent - serviceAccountAnnotations: {} + # -- Default resources requests and limits for Flyteadmin deployment resources: limits: cpu: 250m @@ -19,26 +21,37 @@ flyteadmin: cpu: 10m ephemeral-storage: 50Mi memory: 50Mi + # -- Default regex string for searching configuration files configPath: /etc/flyte/config/*.yaml + # -- Service settings for Flyteadmin service: annotations: contour.heptio.com/upstream-protocol.h2c: grpc + type: ClusterIP + # -- Annotations for ServiceAccount attached to Flyteadmin pods + serviceAccountAnnotations: {} + # -- Annotations for Flyteadmin pods podAnnotations: {} + # -- nodeSelector for Flyteadmin deployment nodeSelector: {} + # -- tolerations for Flyteadmin deployment tolerations: [] + # -- affinity for Flyteadmin deployment affinity: {} # -# DATACATALOG +# DATACATALOG SETTINGS # datacatalog: + # -- Replicas count for Datacatalog deployment replicaCount: 1 image: + # -- Docker image for Datacatalog deployment repository: docker.io/lyft/datacatalog tag: v0.2.2 pullPolicy: IfNotPresent - serviceAccountAnnotations: {} + # -- Default resources requests and limits for Datacatalog deployment resources: limits: cpu: 500m @@ -48,27 +61,37 @@ datacatalog: cpu: 10m ephemeral-storage: 50Mi memory: 50Mi + # -- Default regex string for searching configuration files configPath: /etc/datacatalog/config/*.yaml + # -- Service settings for Datacatalog service: annotations: contour.heptio.com/upstream-protocol.h2c: grpc type: NodePort + # -- Annotations for ServiceAccount attached to Datacatalog pods + serviceAccountAnnotations: {} + # -- Annotations for Datacatalog pods podAnnotations: {} + # -- nodeSelector for Datacatalog deployment nodeSelector: {} + # -- tolerations for Datacatalog deployment tolerations: [] + # -- affinity for Datacatalog deployment affinity: {} # -# FLYTEPROPELLER +# FLYTEPROPELLER SETTINGS # flytepropeller: + # -- Replicas count for Flytepropeller deployment replicaCount: 1 image: + # -- Docker image for Flytepropeller deployment repository: docker.io/lyft/flytepropeller tag: v0.4.2 pullPolicy: IfNotPresent - serviceAccountAnnotations: {} + # -- Default resources requests and limits for Flytepropeller deployment resources: limits: cpu: 200m @@ -78,22 +101,32 @@ flytepropeller: cpu: 10m ephemeral-storage: 50Mi memory: 50Mi + # -- Default regex string for searching configuration files configPath: /etc/flyte/config/*.yaml + # -- Annotations for ServiceAccount attached to Flytepropeller pods + serviceAccountAnnotations: {} + # -- Annotations for Flytepropeller pods podAnnotations: {} + # -- nodeSelector for Flytepropeller deployment nodeSelector: {} + # -- tolerations for Flytepropeller deployment tolerations: [] + # -- affinity for Flytepropeller deployment affinity: {} # -# FLYTECONSOLE +# FLYTECONSOLE SETTINGS # flyteconsole: + # -- Replicas count for Flyteconsole deployment replicaCount: 1 image: + # -- Docker image for Flyteconsole deployment repository: docker.io/lyft/flyteconsole tag: v0.12.1 pullPolicy: IfNotPresent + # -- Default resources requests and limits for Flyteconsole deployment resources: limits: cpu: 500m @@ -101,87 +134,140 @@ flyteconsole: requests: cpu: 10m memory: 50Mi - service: {} + # -- Service settings for Flyteconsole + service: + annotations: {} + type: ClusterIP + # -- Annotations for Flyteconsole pods podAnnotations: {} + # -- nodeSelector for Flyteconsole deployment nodeSelector: {} + # -- tolerations for Flyteconsole deployment tolerations: [] + # -- affinity for Flyteconsole deployment affinity: {} # -# REDIS +# REDIS SETTINGS # redis: - enabled: true # Set false to define malternative redis + # --- enable or disable Redis Statefulset installation + enabled: true + # -- Replicas count for Redis Statefulset replicaCount: 1 image: + # -- Docker image for Redis Statefulset repository: docker.io/bitnami/redis tag: 4.0.2-r1 pullPolicy: IfNotPresent + # -- Default resources requests and limits for Redis Statefulset resources: requests: cpu: 10m memory: 50Mi - service: {} + limits: + cpu: 1000m + memory: 1Gi + # -- Service settings for Redis + service: + annotations: {} + type: ClusterIP + # -- Annotations for Redis pods podAnnotations: {} + # -- nodeSelector for Redis Statefulset nodeSelector: {} + # -- tolerations for Redis Statefulset tolerations: [] + # -- affinity for Redis Statefulset affinity: {} # -# POSTGRES +# POSTGRES SETTINGS # postgres: - enabled: true # Set false if you are going to use RDS + # --- enable or disable Postgres deployment installation + enabled: true + # -- Replicas count for Postgres deployment replicaCount: 1 image: + # -- Docker image for Postgres deployment repository: postgres tag: "10.1" pullPolicy: IfNotPresent + # -- Default resources requests and limits for Postgres deployment resources: requests: cpu: 10m memory: 128Mi - service: {} + limits: + cpu: 1000m + memory: 512Mi + # -- Service settings for Postgres + service: + annotations: {} + type: ClusterIP + # -- Annotations for Postgres pods podAnnotations: {} + # -- nodeSelector for Postgres deployment nodeSelector: {} + # -- tolerations for Postgres deployment tolerations: [] + # -- affinity for Postgres deployment affinity: {} # -# MINIO +# MINIO SETTINGS # minio: - enabled: true # Set false if you'd like using S3 + # --- enable or disable Minio deployment installation + enabled: true + # -- Replicas count for Minio deployment replicaCount: 1 image: + # -- Docker image for Minio deployment repository: minio/minio tag: RELEASE.2019-06-04T01-15-58Z pullPolicy: IfNotPresent + # -- Default resources requests and limits for Minio deployment resources: requests: cpu: 10m memory: 128Mi - service: {} + limits: + cpu: 200m + memory: 512Mi + # -- Service settings for Minio + service: + annotations: {} + type: ClusterIP + # -- Annotations for Minio pods podAnnotations: {} + # -- nodeSelector for Minio deployment nodeSelector: {} + # -- tolerations for Minio deployment tolerations: [] + # -- affinity for Minio deployment affinity: {} # -# CONTOUR +# CONTOUR SETTINGS # contour: - enabled: true # Set false if you have alternative ingress controller like Istio or Ingress-Nginx + # --- enable or disable Contour deployment installation + enabled: true + # -- Replicas count for Contour deployment replicaCount: 1 contour: image: + # -- Docker image for Contour repository: gcr.io/heptio-images/contour tag: v0.6.1 pullPolicy: IfNotPresent + # -- Default resources requests and limits for Contour resources: requests: cpu: 10m @@ -191,9 +277,11 @@ contour: memory: 100Mi envoy: image: + # -- Docker image for Envoy repository: docker.io/envoyproxy/envoy-alpine tag: v1.6.0 pullPolicy: IfNotPresent + # -- Default resources requests and limits for Envoy resources: requests: cpu: 10m @@ -201,33 +289,40 @@ contour: limits: cpu: 100m memory: 100Mi + # -- Service settings for Contour service: - annotations: + annotations: {} type: NodePort ports: - protocol: TCP port: 80 nodePort: 30081 + # -- Annotations for ServiceAccount attached to Contour pods serviceAccountAnnotations: {} + # -- Annotations for Contour pods podAnnotations: {} + # -- nodeSelector for Contour deployment nodeSelector: {} + # -- tolerations for Contour deployment tolerations: [] + # -- affinity for Contour deployment affinity: {} # -# SPARKOPERATOR +# SPARKOPERATOR SETTINGS # sparkoperator: - enabled: true # Set false to disable + # --- enable or disable Sparkoperator deployment installation + enabled: true + # -- Replicas count for Sparkoperator deployment replicaCount: 1 image: + # -- Docker image for Sparkoperator repository: gcr.io/spark-operator/spark-operator tag: v2.4.0-v1beta1-0.9.0 pullPolicy: IfNotPresent - serviceAccountAnnotations: {} - securityContext: {} - service: {} + # -- Default resources requests and limits for Sparkoperator resources: limits: cpu: 1000m @@ -235,24 +330,36 @@ sparkoperator: requests: cpu: 10m memory: 50M + # -- Service settings for Contour + service: + annotations: {} + type: ClusterIP + # -- Annotations for ServiceAccount attached to Sparkoperator pods + serviceAccountAnnotations: {} + # -- Annotations for Sparkoperator pods podAnnotations: {} + # -- nodeSelector for Sparkoperator deployment nodeSelector: {} + # -- tolerations for Sparkoperator deployment tolerations: [] + # -- affinity for Sparkoperator deployment affinity: {} # -# PYTORCHOPERATOR +# PYTORCHOPERATOR SETTINGS # pytorchoperator: + # --- enable or disable Pytorchoperator deployment installation enabled: true # Set false to disable + # -- Replicas count for Pytorchoperator deployment replicaCount: 1 image: + # -- Docker image for Pytorchoperator repository: gcr.io/kubeflow-images-public/pytorch-operator tag: v1.0.0-g047cf0f pullPolicy: IfNotPresent - serviceAccountAnnotations: {} - service: {} + # -- Default resources requests and limits for Pytorchoperator resources: limits: cpu: 500m @@ -260,32 +367,51 @@ pytorchoperator: requests: cpu: 10m memory: 50M + # -- Service settings for Pytorchoperator + service: + annotations: {} + type: ClusterIP + # -- Annotations for ServiceAccount attached to Pytorchoperator pods + serviceAccountAnnotations: {} + # -- Annotations for Pytorchoperator pods podAnnotations: {} + # -- nodeSelector for Pytorchoperator deployment nodeSelector: {} + # -- tolerations for Pytorchoperator deployment tolerations: [] + # -- affinity for Pytorchoperator deployment affinity: {} # -# COMMON +# COMMON SETTINGS # common: databaseSecret: - create: false - name: - ingressAnnotations: {} - # nginx.ingress.kubernetes.io/ssl-redirect: "false" + # -- Specify name of K8s Secret which contains Database password. Leave it empty if you don't need this Secret + name: "" + # -- Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets + secretManifest: {} + ingress: + # --- enable or disable creating Ingress for Flyte. Relevant to disable when using e.g. Istio as ingress controller. + enabled: true + annotations: {} + flyteNamespaceTemplate: + # --- enable or disable creating Flyte namespace in template. Enable whem using helm as template-engine only. Disable when using `helm install ...`. + enabled: true # -# CONFIGMAPS +# CONFIGMAPS SETTINGS # configmap: + # Configmap for Flyte console UI console: BASE_URL: /console CONFIG_DIR: /etc/flyte/config + # Configmap for ClusterResource parameters cluster_resources: cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" @@ -307,9 +433,16 @@ configmap: value: "3000Mi" refresh: 5m + # Database configuration db: database: {} + # port: 5432 + # username: postgres + # host: postgres + # dbname: flyte + # passwordPath: /etc/db/pass.txt + # Domains configuration for Flyte projects domain: domains: - id: development @@ -319,6 +452,7 @@ configmap: - id: production name: production + # Remote data configuration remote_data: remoteData: region: us-east-1 @@ -326,6 +460,7 @@ configmap: signedUrls: durationMinutes: 3 + # Flyteadmin server config server: server: httpPort: 8088 @@ -349,12 +484,14 @@ configmap: testing: host: http://flyteadmin + # Datacatalog server config datacatalogServer: datacatalog: storage-prefix: metadata/datacatalog metrics-scope: datacatalog profiler-port: 10254 + # Storage configuration storage: storage: type: minio @@ -363,6 +500,7 @@ configmap: auth-type: accesskey region: us-east-1 + # Task default resources configuration task_resource_defaults: task_resources: defaults: @@ -374,6 +512,7 @@ configmap: memory: 100Gi storage: 400Gi + # Admin configuration admin: event: type: admin @@ -383,12 +522,14 @@ configmap: endpoint: flyteadmin:81 insecure: true + # Catalog configuration catalog: catalog-cache: endpoint: datacatalog:89 type: datacatalog insecure: true + # Catalog cache configuration catalog_cache: plugins: catalogCache: @@ -397,6 +538,7 @@ configmap: writer: maxItems: 10000 + # Copilot configuration copilot: plugins: k8s: @@ -405,6 +547,7 @@ configmap: image: docker.io/lyft/flytecopilot:v0.3.35 start-timeout: 30s + # Core propeller configuration core: propeller: rawoutput-prefix: s3://my-s3-bucket/ @@ -438,28 +581,36 @@ configmap: rate: 10 capacity: 100 + # Plugins configuration enabled_plugins: tasks: max-plugin-phase-versions: 1000000 task-plugins: enabled-plugins: [] + # Kubernetes plugin configuration k8s: plugins: k8s: + default-env-vars: [] + # - DEFAULT_ENV_VAR: VALUE + default-cpus: 100m default-memory: 100Mi - + + # Logger configuration logger: logger: show-source: true level: 4 + # Qubole plugin configuration qubole: plugins: qubole: quboleTokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN" + # Resource manager configuration resource_manager: propeller: resourcemanager: @@ -469,9 +620,11 @@ configmap: hostPath: redis-resource-manager:6379 hostKey: mypassword + # Spark plugin configuration spark: plugins: spark: + # -- Spark default configuration spark-config-default: - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - spark.kubernetes.allocation.batch.size: "50" @@ -487,11 +640,13 @@ configmap: - spark.blacklist.timeout: "5m" - spark.task.maxfailures: "8" + # Tasks logs plugin configuration task_logs: plugins: logs: - kubernetes-enabled: true + kubernetes-enabled: false + # -- Template for namespaces resources aa_namespace: | apiVersion: v1 kind: Namespace @@ -501,6 +656,7 @@ configmap: finalizers: - kubernetes + # -- Template for spark role ad_spark_role: | apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role @@ -527,6 +683,7 @@ configmap: verbs: - '*' + # -- Template for spark SA ae_spark_service_account: | apiVersion: v1 kind: ServiceAccount @@ -534,6 +691,7 @@ configmap: name: spark namespace: {{ namespace }} + # -- Template for spark role binding af_spark_role_binding: | apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding From 565f0b4fe2310179da373177ca460123c6b1a113 Mon Sep 17 00:00:00 2001 From: Ruslan Stanevich Date: Mon, 9 Nov 2020 18:18:16 +0300 Subject: [PATCH 05/92] improve templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/admin/rbac.yaml | 2 +- helm/templates/common/namespace.yaml | 3 +++ helm/templates/contour/namespace.yaml | 2 +- helm/templates/sparkoperator/deployment.yaml | 7 ++++--- helm/templates/sparkoperator/job.yaml | 6 ++++-- helm/templates/sparkoperator/service.yaml | 2 +- helm/values.yaml | 14 ++++++++------ 7 files changed, 22 insertions(+), 14 deletions(-) diff --git a/helm/templates/admin/rbac.yaml b/helm/templates/admin/rbac.yaml index 8b430aca9c..9696437360 100644 --- a/helm/templates/admin/rbac.yaml +++ b/helm/templates/admin/rbac.yaml @@ -40,7 +40,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: {{ template "flyteadmin.name" . }} + name: {{ template "flyteadmin.name" . }}-binding labels: {{ include "flyteadmin.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io diff --git a/helm/templates/common/namespace.yaml b/helm/templates/common/namespace.yaml index c8a3dc17a7..7e4f26d461 100644 --- a/helm/templates/common/namespace.yaml +++ b/helm/templates/common/namespace.yaml @@ -3,4 +3,7 @@ apiVersion: v1 kind: Namespace metadata: name: {{ template "flyte.namespace" . }} +spec: + finalizers: + - kubernetes {{- end }} diff --git a/helm/templates/contour/namespace.yaml b/helm/templates/contour/namespace.yaml index 5262f4fb84..a7148f0bfb 100644 --- a/helm/templates/contour/namespace.yaml +++ b/helm/templates/contour/namespace.yaml @@ -1,4 +1,4 @@ -{{- if .Values.sparkoperator.enabled }} +{{- if .Values.contour.enabled }} apiVersion: v1 kind: Namespace metadata: diff --git a/helm/templates/sparkoperator/deployment.yaml b/helm/templates/sparkoperator/deployment.yaml index 07734eb29d..6464e1f99a 100644 --- a/helm/templates/sparkoperator/deployment.yaml +++ b/helm/templates/sparkoperator/deployment.yaml @@ -23,15 +23,16 @@ spec: command: - /usr/bin/spark-operator args: + - -webhook-svc-namespace + - sparkoperator - -logtostderr - -v=2 - - -controller-threads=20 + - -enable-webhook=true + - -controller-threads=5 - -enable-metrics=true - "-metrics-prefix=service:" - -metrics-labels=task_name - -metrics-labels=workflow_name - - -enable-webhook=true - - -webhook-svc-namespace=sparkoperator env: - name: GOMAXPROCS value: "16" diff --git a/helm/templates/sparkoperator/job.yaml b/helm/templates/sparkoperator/job.yaml index fc155d54cb..e3129866f5 100644 --- a/helm/templates/sparkoperator/job.yaml +++ b/helm/templates/sparkoperator/job.yaml @@ -17,12 +17,14 @@ spec: containers: - image: "{{ .Values.sparkoperator.image.repository }}:{{ .Values.sparkoperator.image.tag }}" imagePullPolicy: "{{ .Values.sparkoperator.image.pullPolicy }}" - name: sparkoperator + name: main command: - /usr/bin/gencerts.sh - --namespace - sparkoperator - - -p + - '--service' + - spark-webhook + - '--in-pod' restartPolicy: Never serviceAccountName: {{ template "sparkoperator.name" . }} {{- end }} diff --git a/helm/templates/sparkoperator/service.yaml b/helm/templates/sparkoperator/service.yaml index 90840e8611..80da0c9359 100644 --- a/helm/templates/sparkoperator/service.yaml +++ b/helm/templates/sparkoperator/service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "sparkoperator.name" . }} + name: spark-webhook namespace: {{ template "sparkoperator.namespace" . }} labels: {{ include "sparkoperator.labels" . | nindent 4 }} {{- with .Values.sparkoperator.service.annotations }} diff --git a/helm/values.yaml b/helm/values.yaml index 5c6d1ca33a..25ccb54a3e 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -413,6 +413,7 @@ configmap: # Configmap for ClusterResource parameters cluster_resources: + # -- ClusterResource parameters cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" customData: @@ -460,7 +461,7 @@ configmap: signedUrls: durationMinutes: 3 - # Flyteadmin server config + # -- Flyteadmin server config server: server: httpPort: 8088 @@ -469,11 +470,6 @@ configmap: secure: false useAuth: false allowCors: true - allowedOrigins: - # Accepting all domains for Sandbox installation - - "*" - allowedHeaders: - - "Content-Type" flyteadmin: roleNameKey: "iam.amazonaws.com/role" profilerPort: 10254 @@ -490,6 +486,10 @@ configmap: storage-prefix: metadata/datacatalog metrics-scope: datacatalog profiler-port: 10254 + application: + grpcPort: 8089 + httpPort: 8080 + grpcServerReflection: true # Storage configuration storage: @@ -502,6 +502,7 @@ configmap: # Task default resources configuration task_resource_defaults: + # -- Task default resources parameters task_resources: defaults: cpu: 100m @@ -612,6 +613,7 @@ configmap: # Resource manager configuration resource_manager: + # -- resource manager configuration propeller: resourcemanager: type: redis From 8ec2cab4166311e1f79f383cac23159012eb8a34 Mon Sep 17 00:00:00 2001 From: Ruslan Stanevich Date: Tue, 2 Feb 2021 21:23:33 +0300 Subject: [PATCH 06/92] Support gRPC loadbalncing for EKS with AWS ALB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/common/ingress.yaml | 72 ++++++++++++++++++++++++++++++ helm/values-eks.yaml | 65 +++++---------------------- 2 files changed, 83 insertions(+), 54 deletions(-) diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index 67539b0658..6bbf9f10e7 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -1,4 +1,7 @@ {{- if .Values.common.ingress.enabled }} + +{{- if .Values.contour.enabled }} +--- apiVersion: extensions/v1beta1 kind: Ingress metadata: @@ -39,4 +42,73 @@ spec: serviceName: flyteadmin servicePort: 87 path: /openapi + +{{- else }} +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "flyte.name" . }}-http1 + namespace: {{ template "flyte.namespace" . }} + annotations: + alb.ingress.kubernetes.io/backend-protocol-version: HTTP1 + {{- with .Values.common.ingress.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + rules: + - {{- with .Values.common.ingress.host }} + host: {{ . }} + {{- end }} + http: + paths: + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /console* + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /__webpack_hmr* + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /api* + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /healthcheck* + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /v1* + - backend: + serviceName: flyteadmin + servicePort: 87 + path: /openapi* +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "flyte.name" . }}-grpc + namespace: {{ template "flyte.namespace" . }} + annotations: + alb.ingress.kubernetes.io/backend-protocol-version: GRPC + {{- with .Values.common.ingress.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} +spec: + rules: + - {{- with .Values.common.ingress.host }} + host: {{ . }} + {{- end }} + http: + paths: + - backend: + serviceName: flyteadmin + servicePort: 81 + path: /flyteidl.service.AdminService* + +{{- end }} + {{- end }} \ No newline at end of file diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 0059572e80..5dbb89468a 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -169,60 +169,7 @@ minio: # contour: - enabled: true - replicaCount: 2 - contour: - image: - repository: gcr.io/heptio-images/contour - tag: v0.6.1 - pullPolicy: IfNotPresent - resources: - requests: - cpu: 50m - memory: 100Mi - limits: - cpu: 2000m - memory: 1000Mi - envoy: - image: - repository: docker.io/envoyproxy/envoy-alpine - tag: v1.6.0 - pullPolicy: IfNotPresent - resources: - requests: - cpu: 50m - memory: 100Mi - limits: - cpu: 2000m - memory: 1000Mi - - service: - annotations: - external-dns.alpha.kubernetes.io/hostname: flyte.examlple.com - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-east-1::certificate/" - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp" - service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: "*" - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "3600" - service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" - service.beta.kubernetes.io/aws-load-balancer-internal: "true" - service.beta.kubernetes.io/aws-load-balancer-type: nlb - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443" - type: LoadBalancer - ports: - - name: tcp - protocol: TCP - port: 80 - - name: tls - protocol: TCP - port: 443 - targetPort: 80 - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app.kubernetes.io/name: contour - topologyKey: kubernetes.io/hostname + enabled: false # # SPARKOPERATOR @@ -267,6 +214,16 @@ pytorchoperator: # common: + ingress: + host: flyte.example.com + annotations: + # aws-load-balancer-controller v2.1 or higher is required - https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.1/ + kubernetes.io/ingress.class: alb + alb.ingress.kubernetes.io/group.name: flyte-ingress + alb.ingress.kubernetes.io/scheme: internal + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/backend-protocol: HTTP + alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS": 443}]' databaseSecret: name: db-pass secretManifest: From 249711f29df577fc6348b38f64356a6d9cb18834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Tue, 23 Feb 2021 11:16:28 +0100 Subject: [PATCH 07/92] Use external chart dependencies for contour and spark-operator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump versions Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/Chart.yaml | 14 +- helm/README.MD | 2 +- helm/templates/_helpers.tpl | 41 ----- .../templates/contour/crds/ingressroutes.yaml | 136 -------------- helm/templates/contour/deployment.yaml | 83 --------- helm/templates/contour/namespace.yaml | 6 - helm/templates/contour/rbac.yaml | 78 -------- helm/templates/contour/service.yaml | 17 -- .../crds/scheduledsparkapplications.yaml | 103 ----------- .../sparkoperator/crds/sparkapplications.yaml | 149 ---------------- helm/templates/sparkoperator/deployment.yaml | 60 ------- helm/templates/sparkoperator/job.yaml | 30 ---- helm/templates/sparkoperator/namespace.yaml | 6 - helm/templates/sparkoperator/rbac.yaml | 167 ------------------ helm/templates/sparkoperator/service.yaml | 20 --- helm/values-sandbox.yaml | 10 +- helm/values.yaml | 62 ++----- 17 files changed, 37 insertions(+), 947 deletions(-) delete mode 100644 helm/templates/contour/crds/ingressroutes.yaml delete mode 100644 helm/templates/contour/deployment.yaml delete mode 100644 helm/templates/contour/namespace.yaml delete mode 100644 helm/templates/contour/rbac.yaml delete mode 100644 helm/templates/contour/service.yaml delete mode 100644 helm/templates/sparkoperator/crds/scheduledsparkapplications.yaml delete mode 100644 helm/templates/sparkoperator/crds/sparkapplications.yaml delete mode 100644 helm/templates/sparkoperator/deployment.yaml delete mode 100644 helm/templates/sparkoperator/job.yaml delete mode 100644 helm/templates/sparkoperator/namespace.yaml delete mode 100644 helm/templates/sparkoperator/rbac.yaml delete mode 100644 helm/templates/sparkoperator/service.yaml diff --git a/helm/Chart.yaml b/helm/Chart.yaml index aef2a99bd4..b6bb75125a 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,5 +2,17 @@ apiVersion: v2 name: flyte description: A Helm chart for Flyte type: application -version: 0.1.0 +version: 0.2.0 appVersion: 0.8.0 +dependencies: + - name: contour + version: 4.1.2 + repository: https://charts.bitnami.com/bitnami + condition: contour.enabled + - name: spark-operator + alias: sparkoperator + version: 1.0.6 + repository: https://googlecloudplatform.github.io/spark-on-k8s-operator + condition: spark.enabled + + diff --git a/helm/README.MD b/helm/README.MD index 26a6dc142b..227b37c1a7 100644 --- a/helm/README.MD +++ b/helm/README.MD @@ -5,7 +5,7 @@ A Helm chart for Flyte ### SANDBOX INSTALLATION: -- Install `helm 3` realese version +- Install `helm 3` release version - Generate `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` - Deploy the manifest `kubectl apply -f flyte_generated_sandbox.yaml` - When all pods are running - run end2end tests: `kubectl apply -f ../end2end/tests/endtoend.yaml` diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 398d189543..244619bbad 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -124,47 +124,6 @@ helm.sh/chart: {{ include "flyte.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} - -{{- define "contour.name" -}} -contour -{{- end -}} - -{{- define "contour.namespace" -}} -heptio-contour -{{- end -}} - -{{- define "contour.selectorLabels" -}} -app.kubernetes.io/name: {{ template "contour.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} - -{{- define "contour.labels" -}} -{{ include "contour.selectorLabels" . }} -helm.sh/chart: {{ include "flyte.chart" . }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - - -{{- define "sparkoperator.name" -}} -sparkoperator -{{- end -}} - -{{- define "sparkoperator.namespace" -}} -sparkoperator -{{- end -}} - -{{- define "sparkoperator.selectorLabels" -}} -app.kubernetes.io/name: {{ template "sparkoperator.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} - -{{- define "sparkoperator.labels" -}} -{{ include "sparkoperator.selectorLabels" . }} -helm.sh/chart: {{ include "flyte.chart" . }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - - {{- define "pytorch-operator.name" -}} pytorch-operator {{- end -}} diff --git a/helm/templates/contour/crds/ingressroutes.yaml b/helm/templates/contour/crds/ingressroutes.yaml deleted file mode 100644 index d4b258c836..0000000000 --- a/helm/templates/contour/crds/ingressroutes.yaml +++ /dev/null @@ -1,136 +0,0 @@ -{{- if .Values.contour.enabled }} -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ingressroutes.contour.heptio.com - labels: - component: ingressroute -spec: - group: contour.heptio.com - version: v1beta1 - scope: Namespaced - names: - plural: ingressroutes - kind: IngressRoute - additionalPrinterColumns: - - name: FQDN - type: string - description: Fully qualified domain name - JSONPath: .spec.virtualhost.fqdn - - name: TLS Secret - type: string - description: Secret with TLS credentials - JSONPath: .spec.virtualhost.tls.secretName - - name: First route - type: string - description: First routes defined - JSONPath: .spec.routes[0].match - - name: Status - type: string - description: The current status of the IngressRoute - JSONPath: .status.currentStatus - - name: Status Description - type: string - description: Description of the current status - JSONPath: .status.description - validation: - openAPIV3Schema: - properties: - spec: - required: - - routes - properties: - virtualhost: - properties: - fqdn: - type: string - pattern: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-z]{2,}$ - aliases: - type: array - items: - type: string - pattern: ^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$ - strategy: - type: string - enum: - - RoundRobin - - LeastRequest - - Random - - RingHash - - Maglev - healthCheck: - type: object - required: - - path - properties: - path: - type: string - pattern: ^\/.*$ - intervalSeconds: - type: integer - timeoutSeconds: - type: integer - unhealthyThresholdCount: - type: integer - healthyThresholdCount: - type: integer - routes: - type: array - items: - required: - - match - properties: - match: - type: string - pattern: ^\/.*$ - delegate: - type: object - required: - - name - properties: - name: - type: string - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 - namespace: - type: string - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 - services: - type: array - items: - type: object - required: - - name - - port - properties: - name: - type: string - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 - port: - type: integer - weight: - type: integer - strategy: - type: string - enum: - - RoundRobin - - WeightedLeastRequest - - Random - - RingHash - - Maglev - healthCheck: - type: object - required: - - path - properties: - path: - type: string - pattern: ^\/.*$ - intervalSeconds: - type: integer - timeoutSeconds: - type: integer - unhealthyThresholdCount: - type: integer - healthyThresholdCount: - type: integer -{{- end }} diff --git a/helm/templates/contour/deployment.yaml b/helm/templates/contour/deployment.yaml deleted file mode 100644 index d4733d7257..0000000000 --- a/helm/templates/contour/deployment.yaml +++ /dev/null @@ -1,83 +0,0 @@ -{{- if .Values.contour.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "contour.name" . }} - namespace: {{ template "contour.namespace" . }} - labels: {{ include "contour.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.contour.replicaCount }} - selector: - matchLabels: {{ include "contour.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.contour.podAnnotations }} - annotations: {{ toYaml . | nindent 8 }} - {{- end }} - labels: {{ include "contour.labels" . | nindent 8 }} - spec: - initContainers: - - image: "{{ .Values.contour.contour.image.repository }}:{{ .Values.contour.contour.image.tag }}" - imagePullPolicy: "{{ .Values.contour.contour.image.pullPolicy }}" - name: envoy-initconfig - command: - - contour - args: - - bootstrap - - /config/contour.yaml - - --statsd-enabled - volumeMounts: - - name: contour-config - mountPath: /config - containers: - - image: "{{ .Values.contour.envoy.image.repository }}:{{ .Values.contour.envoy.image.tag }}" - imagePullPolicy: "{{ .Values.contour.envoy.image.pullPolicy }}" - name: envoy - command: - - envoy - args: - - -c - - /config/contour.yaml - - --service-cluster - - cluster0 - - --service-node - - node0 - ports: - - containerPort: 80 - name: http - - containerPort: 8002 - name: statsd - resources: {{ toYaml .Values.contour.envoy.resources | nindent 10 }} - volumeMounts: - - name: contour-config - mountPath: /config - - image: "{{ .Values.contour.contour.image.repository }}:{{ .Values.contour.contour.image.tag }}" - imagePullPolicy: "{{ .Values.contour.contour.image.pullPolicy }}" - ports: - - containerPort: 8000 - name: contour - name: contour - command: - - contour - args: - - serve - - --incluster - - --envoy-http-port=80 - - --debug-http-port=6069 - resources: {{ toYaml .Values.contour.contour.resources | nindent 10 }} - volumes: - - name: contour-config - emptyDir: {} - dnsPolicy: ClusterFirst - serviceAccountName: {{ template "contour.name" . }} - terminationGracePeriodSeconds: 30 - {{- with .Values.contour.nodeSelector }} - nodeSelector: {{ toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.contour.affinity }} - affinity: {{ toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.contour.tolerations }} - tolerations: {{ toYaml . | nindent 8 }} - {{- end }} -{{- end }} diff --git a/helm/templates/contour/namespace.yaml b/helm/templates/contour/namespace.yaml deleted file mode 100644 index a7148f0bfb..0000000000 --- a/helm/templates/contour/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{{- if .Values.contour.enabled }} -apiVersion: v1 -kind: Namespace -metadata: - name: {{ template "contour.namespace" . }} -{{- end }} diff --git a/helm/templates/contour/rbac.yaml b/helm/templates/contour/rbac.yaml deleted file mode 100644 index 00d6d43868..0000000000 --- a/helm/templates/contour/rbac.yaml +++ /dev/null @@ -1,78 +0,0 @@ -{{- if .Values.contour.enabled }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ template "contour.name" . }} - namespace: {{ template "contour.namespace" . }} - labels: {{ include "contour.labels" . | nindent 4 }} - {{- with .Values.contour.serviceAccountAnnotations }} - annotations: {{ toYaml . | nindent 4 }} - {{- end}} - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ template "contour.name" . }} - labels: {{ include "contour.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: ["contour.heptio.com"] - resources: ["ingressroutes"] - verbs: - - get - - list - - watch - - put - - post - - patch - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: {{ template "contour.name" . }} - labels: {{ include "contour.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ template "contour.name" . }} -subjects: -- kind: ServiceAccount - name: {{ template "contour.name" . }} - namespace: {{ template "contour.namespace" . }} - -{{- end }} diff --git a/helm/templates/contour/service.yaml b/helm/templates/contour/service.yaml deleted file mode 100644 index e64e19d6ff..0000000000 --- a/helm/templates/contour/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.contour.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ template "contour.name" . }} - namespace: {{ template "contour.namespace" . }} - labels: {{ include "contour.labels" . | nindent 4 }} - {{- with .Values.contour.service.annotations }} - annotations: {{ toYaml . | nindent 4 }} - {{- end }} -spec: - {{- with .Values.contour.service.type}} - type: {{ . }} - {{- end }} - ports: {{ toYaml .Values.contour.service.ports | nindent 4 }} - selector: {{ include "contour.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/helm/templates/sparkoperator/crds/scheduledsparkapplications.yaml b/helm/templates/sparkoperator/crds/scheduledsparkapplications.yaml deleted file mode 100644 index 6df8778411..0000000000 --- a/helm/templates/sparkoperator/crds/scheduledsparkapplications.yaml +++ /dev/null @@ -1,103 +0,0 @@ -{{- if .Values.sparkoperator.enabled }} -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: scheduledsparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: ScheduledSparkApplication - listKind: ScheduledSparkApplicationList - plural: scheduledsparkapplications - shortNames: - - scheduledsparkapp - singular: scheduledsparkapplication - scope: Namespaced - validation: - openAPIV3Schema: - properties: - spec: - properties: - concurrencyPolicy: - enum: - - Allow - - Forbid - - Replace - failedRunHistoryLimit: - minimum: 1 - type: integer - schedule: - type: string - successfulRunHistoryLimit: - minimum: 1 - type: integer - template: - properties: - deps: - properties: - downloadTimeout: - minimum: 1 - type: integer - maxSimultaneousDownloads: - minimum: 1 - type: integer - driver: - properties: - cores: - exclusiveMinimum: true - minimum: 0 - type: number - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - executor: - properties: - cores: - exclusiveMinimum: true - minimum: 0 - type: number - instances: - minimum: 1 - type: integer - mode: - enum: - - cluster - - client - monitoring: - properties: - prometheus: - properties: - port: - maximum: 49151 - minimum: 1024 - type: integer - pythonVersion: - enum: - - "2" - - "3" - restartPolicy: - properties: - onFailureRetries: - minimum: 0 - type: integer - onFailureRetryInterval: - minimum: 1 - type: integer - onSubmissionFailureRetries: - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - minimum: 1 - type: integer - type: - enum: - - Never - - OnFailure - - Always - type: - enum: - - Java - - Scala - - Python - - R - version: v1beta1 -{{- end }} diff --git a/helm/templates/sparkoperator/crds/sparkapplications.yaml b/helm/templates/sparkoperator/crds/sparkapplications.yaml deleted file mode 100644 index c4bed90488..0000000000 --- a/helm/templates/sparkoperator/crds/sparkapplications.yaml +++ /dev/null @@ -1,149 +0,0 @@ -{{- if .Values.sparkoperator.enabled }} -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: sparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: SparkApplication - listKind: SparkApplicationList - plural: sparkapplications - shortNames: - - sparkapp - singular: sparkapplication - scope: Namespaced - validation: - openAPIV3Schema: - properties: - metadata: - properties: - name: - maxLength: 63 - minLength: 1 - type: string - spec: - properties: - arguments: - items: - type: string - type: array - deps: - properties: - downloadTimeout: - minimum: 1 - type: integer - maxSimultaneousDownloads: - minimum: 1 - type: integer - driver: - properties: - cores: - exclusiveMinimum: true - minimum: 0 - type: number - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - executor: - properties: - cores: - exclusiveMinimum: true - minimum: 0 - type: number - instances: - minimum: 1 - type: integer - failureRetries: - type: integer - hadoopConf: - type: object - hadoopConfigMap: - type: string - image: - type: string - imagePullPolicy: - enum: - - Always - - Never - - IfNotPresent - imagePullSecrets: - items: - type: string - type: array - initContainerImage: - type: string - mainApplicationFile: - type: string - mainClass: - type: string - memoryOverheadFactor: - type: string - mode: - enum: - - cluster - - client - monitoring: - properties: - exposeDriverMetrics: - type: boolean - exposeExecutorMetrics: - type: boolean - metricsProperties: - type: string - prometheus: - properties: - port: - maximum: 49151 - minimum: 1024 - type: integer - nodeSelector: - type: object - pythonVersion: - enum: - - "2" - - "3" - restartPolicy: - properties: - onFailureRetries: - minimum: 0 - type: integer - onFailureRetryInterval: - minimum: 1 - type: integer - onSubmissionFailureRetries: - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - minimum: 1 - type: integer - type: - enum: - - Never - - OnFailure - - Always - retryInterval: - type: integer - sparkConf: - type: object - sparkConfigMap: - type: string - sparkVersion: - type: string - type: - enum: - - Java - - Scala - - Python - - R - volumes: - items: - properties: - name: - type: string - type: object - type: array - required: - - type - - sparkVersion - version: v1beta1 -{{- end }} diff --git a/helm/templates/sparkoperator/deployment.yaml b/helm/templates/sparkoperator/deployment.yaml deleted file mode 100644 index 6464e1f99a..0000000000 --- a/helm/templates/sparkoperator/deployment.yaml +++ /dev/null @@ -1,60 +0,0 @@ -{{- if .Values.sparkoperator.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "sparkoperator.name" . }} - namespace: {{ template "sparkoperator.namespace" . }} - labels: {{ include "sparkoperator.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.sparkoperator.replicaCount }} - selector: - matchLabels: {{ include "sparkoperator.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.sparkoperator.podAnnotations }} - annotations: {{ toYaml . | nindent 8 }} - {{- end }} - labels: {{ include "sparkoperator.labels" . | nindent 8 }} - spec: - containers: - - image: "{{ .Values.sparkoperator.image.repository }}:{{ .Values.sparkoperator.image.tag }}" - imagePullPolicy: "{{ .Values.sparkoperator.image.pullPolicy }}" - name: sparkoperator - command: - - /usr/bin/spark-operator - args: - - -webhook-svc-namespace - - sparkoperator - - -logtostderr - - -v=2 - - -enable-webhook=true - - -controller-threads=5 - - -enable-metrics=true - - "-metrics-prefix=service:" - - -metrics-labels=task_name - - -metrics-labels=workflow_name - env: - - name: GOMAXPROCS - value: "16" - ports: - - containerPort: 10254 - - containerPort: 8080 - resources: {{ toYaml .Values.sparkoperator.resources | nindent 10 }} - volumeMounts: - - name: webhook-certs - mountPath: /etc/webhook-certs - serviceAccountName: {{ template "sparkoperator.name" . }} - volumes: - - name: webhook-certs - secret: - secretName: spark-webhook-certs - {{- with .Values.sparkoperator.nodeSelector }} - nodeSelector: {{ toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.sparkoperator.affinity }} - affinity: {{ toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.sparkoperator.tolerations }} - tolerations: {{ toYaml . | nindent 8 }} - {{- end }} -{{- end }} diff --git a/helm/templates/sparkoperator/job.yaml b/helm/templates/sparkoperator/job.yaml deleted file mode 100644 index e3129866f5..0000000000 --- a/helm/templates/sparkoperator/job.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{- if .Values.sparkoperator.enabled }} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ template "sparkoperator.name" . }}-init - namespace: {{ template "sparkoperator.namespace" . }} - labels: {{ include "sparkoperator.labels" . | nindent 4 }} -spec: - backoffLimit: 3 - template: - metadata: - {{- with .Values.sparkoperator.podAnnotations }} - annotations: {{ toYaml . | nindent 8 }} - {{- end }} - labels: {{ include "sparkoperator.labels" . | nindent 8 }} - spec: - containers: - - image: "{{ .Values.sparkoperator.image.repository }}:{{ .Values.sparkoperator.image.tag }}" - imagePullPolicy: "{{ .Values.sparkoperator.image.pullPolicy }}" - name: main - command: - - /usr/bin/gencerts.sh - - --namespace - - sparkoperator - - '--service' - - spark-webhook - - '--in-pod' - restartPolicy: Never - serviceAccountName: {{ template "sparkoperator.name" . }} -{{- end }} diff --git a/helm/templates/sparkoperator/namespace.yaml b/helm/templates/sparkoperator/namespace.yaml deleted file mode 100644 index b4b2ad7dda..0000000000 --- a/helm/templates/sparkoperator/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{{- if .Values.sparkoperator.enabled }} -apiVersion: v1 -kind: Namespace -metadata: - name: {{ template "sparkoperator.namespace" . }} -{{- end }} diff --git a/helm/templates/sparkoperator/rbac.yaml b/helm/templates/sparkoperator/rbac.yaml deleted file mode 100644 index 7a6e8c6083..0000000000 --- a/helm/templates/sparkoperator/rbac.yaml +++ /dev/null @@ -1,167 +0,0 @@ -{{- if .Values.sparkoperator.enabled }} - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ template "sparkoperator.name" . }} - namespace: {{ template "sparkoperator.namespace" . }} - labels: {{ include "sparkoperator.labels" . | nindent 4 }} - {{- with .Values.sparkoperator.serviceAccountAnnotations }} - annotations: {{ toYaml . | nindent 4 }} - {{- end}} - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ template "sparkoperator.name" . }} - labels: {{ include "sparkoperator.labels" . | nindent 4 }} -rules: -- apiGroups: - - "" - resources: - - resourcequotas - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - - secrets - verbs: - - create - - get - - delete -- apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - create - - get - - update - - delete -- apiGroups: - - "" - resources: - - pods - verbs: - - '*' -- apiGroups: - - "" - resources: - - services - - configmaps - - secrets - verbs: - - create - - get - - delete -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - create - - get - - delete -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - get - - update - - delete - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - update - - delete -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - verbs: - - create - - get - - update - - delete -- apiGroups: - - sparkoperator.k8s.io - resources: - - sparkapplications - - scheduledsparkapplications - verbs: - - '*' - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ template "sparkoperator.name" . }}-aggregate-to-admin - labels: {{ include "sparkoperator.labels" . | nindent 4 }} - rbac.authorization.k8s.io/aggregate-to-admin: "true" -rules: -- apiGroups: - - sparkoperator.k8s.io - resources: - - sparkapplications - - scheduledsparkapplications - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: {{ template "sparkoperator.name" . }} - labels: {{ include "sparkoperator.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ template "sparkoperator.name" . }} -subjects: -- kind: ServiceAccount - name: {{ template "sparkoperator.name" . }} - namespace: {{ template "sparkoperator.namespace" . }} - -{{- end }} diff --git a/helm/templates/sparkoperator/service.yaml b/helm/templates/sparkoperator/service.yaml deleted file mode 100644 index 80da0c9359..0000000000 --- a/helm/templates/sparkoperator/service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.sparkoperator.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: spark-webhook - namespace: {{ template "sparkoperator.namespace" . }} - labels: {{ include "sparkoperator.labels" . | nindent 4 }} - {{- with .Values.sparkoperator.service.annotations }} - annotations: {{ toYaml . | nindent 4 }} - {{- end }} -spec: - {{- with .Values.sparkoperator.service.type}} - type: {{ . }} - {{- end }} - ports: - - port: 443 - targetPort: 8080 - name: webhook - selector: {{ include "sparkoperator.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index e5926b4b11..2a625bd026 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -45,7 +45,15 @@ minio: {} # CONTOUR # -contour: {} +contour: + contour: {} + envoy: + service: + type: NodePort + ports: + http: 80 + nodePorts: + http: 30081 # # SPARKOPERATOR diff --git a/helm/values.yaml b/helm/values.yaml index 25ccb54a3e..25583bce4f 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -8,8 +8,8 @@ flyteadmin: replicaCount: 1 image: # -- Docker image for Flyteadmin deployment - repository: docker.io/lyft/flyteadmin - tag: v0.3.7 + repository: ghcr.io/lyft/flyteadmin + tag: v0.3.29 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteadmin deployment resources: @@ -26,7 +26,7 @@ flyteadmin: # -- Service settings for Flyteadmin service: annotations: - contour.heptio.com/upstream-protocol.h2c: grpc + projectcontour.io/upstream-protocol.h2c: grpc type: ClusterIP # -- Annotations for ServiceAccount attached to Flyteadmin pods serviceAccountAnnotations: {} @@ -48,8 +48,8 @@ datacatalog: replicaCount: 1 image: # -- Docker image for Datacatalog deployment - repository: docker.io/lyft/datacatalog - tag: v0.2.2 + repository: ghcr.io/lyft/datacatalog + tag: v0.2.130 pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment resources: @@ -88,8 +88,8 @@ flytepropeller: replicaCount: 1 image: # -- Docker image for Flytepropeller deployment - repository: docker.io/lyft/flytepropeller - tag: v0.4.2 + repository: ghcr.io/flyteorg/flytepropeller + tag: v0.5.13 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -123,8 +123,8 @@ flyteconsole: replicaCount: 1 image: # -- Docker image for Flyteconsole deployment - repository: docker.io/lyft/flyteconsole - tag: v0.12.1 + repository: ghcr.io/lyft/flyteconsole + tag: v0.19.1 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment resources: @@ -229,7 +229,7 @@ minio: image: # -- Docker image for Minio deployment repository: minio/minio - tag: RELEASE.2019-06-04T01-15-58Z + tag: RELEASE.2020-12-16T05-05-17Z pullPolicy: IfNotPresent # -- Default resources requests and limits for Minio deployment resources: @@ -262,11 +262,6 @@ contour: # -- Replicas count for Contour deployment replicaCount: 1 contour: - image: - # -- Docker image for Contour - repository: gcr.io/heptio-images/contour - tag: v0.6.1 - pullPolicy: IfNotPresent # -- Default resources requests and limits for Contour resources: requests: @@ -276,11 +271,6 @@ contour: cpu: 100m memory: 100Mi envoy: - image: - # -- Docker image for Envoy - repository: docker.io/envoyproxy/envoy-alpine - tag: v1.6.0 - pullPolicy: IfNotPresent # -- Default resources requests and limits for Envoy resources: requests: @@ -289,14 +279,6 @@ contour: limits: cpu: 100m memory: 100Mi - # -- Service settings for Contour - service: - annotations: {} - type: NodePort - ports: - - protocol: TCP - port: 80 - nodePort: 30081 # -- Annotations for ServiceAccount attached to Contour pods serviceAccountAnnotations: {} # -- Annotations for Contour pods @@ -319,9 +301,7 @@ sparkoperator: replicaCount: 1 image: # -- Docker image for Sparkoperator - repository: gcr.io/spark-operator/spark-operator - tag: v2.4.0-v1beta1-0.9.0 - pullPolicy: IfNotPresent + tag: v1beta2-1.2.0-3.0.0 # Set to v1beta2-1.1.2-2.4.5 for Spark 2 # -- Default resources requests and limits for Sparkoperator resources: limits: @@ -330,20 +310,6 @@ sparkoperator: requests: cpu: 10m memory: 50M - # -- Service settings for Contour - service: - annotations: {} - type: ClusterIP - # -- Annotations for ServiceAccount attached to Sparkoperator pods - serviceAccountAnnotations: {} - # -- Annotations for Sparkoperator pods - podAnnotations: {} - # -- nodeSelector for Sparkoperator deployment - nodeSelector: {} - # -- tolerations for Sparkoperator deployment - tolerations: [] - # -- affinity for Sparkoperator deployment - affinity: {} # # PYTORCHOPERATOR SETTINGS @@ -397,8 +363,8 @@ common: enabled: true annotations: {} flyteNamespaceTemplate: - # --- enable or disable creating Flyte namespace in template. Enable whem using helm as template-engine only. Disable when using `helm install ...`. - enabled: true + # --- enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. + enabled: false # # CONFIGMAPS SETTINGS @@ -545,7 +511,7 @@ configmap: k8s: co-pilot: name: flyte-copilot- - image: docker.io/lyft/flytecopilot:v0.3.35 + image: ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2 start-timeout: 30s # Core propeller configuration From 6ca7ef6da0097ac532abaecc9d4d010031e5397d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Fri, 19 Mar 2021 23:45:43 +0100 Subject: [PATCH 08/92] Helm - Update task plugin config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize changes: - cd4a3d5a Trim sandbox (#727) - 9b59ac68 Update propeller version and config (#580) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/values-eks.yaml | 6 ++++++ helm/values-gcp.yaml | 20 ++++++++++++++++++++ helm/values-sandbox.yaml | 13 +++++++------ 3 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 helm/values-gcp.yaml diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 5dbb89468a..175dd2fcc4 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -468,6 +468,12 @@ configmap: - spark - k8s-array - pytorch + default-for-task-types: + container: container + sidecar: sidecar + spark: spark + container_array: k8s-array + pytorch: pytorch k8s: plugins: diff --git a/helm/values-gcp.yaml b/helm/values-gcp.yaml new file mode 100644 index 0000000000..31d305c7a1 --- /dev/null +++ b/helm/values-gcp.yaml @@ -0,0 +1,20 @@ +# +# CONFIGMAPS +# + +configmap: + tasks: + max-plugin-phase-versions: 1000000 + task-plugins: + enabled-plugins: + - container + - sidecar + - spark + - k8s-array + - pytorch + default-for-task-types: + container: container + sidecar: sidecar + spark: spark + container_array: k8s-array + pytorch: pytorch \ No newline at end of file diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index 2a625bd026..148d564ba3 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -103,14 +103,15 @@ configmap: enabled_plugins: tasks: - max-plugin-phase-versions: 1000000 task-plugins: enabled-plugins: - - container - - sidecar - - spark - - k8s-array - - pytorch + - container + - sidecar + - k8s-array + default-for-task-types: + container: container + sidecar: sidecar + container_array: k8s-array k8s: plugins: From 11402cda36b71fb18686ff8409159ebbd9074532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Fri, 19 Mar 2021 23:51:29 +0100 Subject: [PATCH 09/92] Helm - Updating Spark config to force default credentials chain for AWS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize change: - 0e2dd723 Updating Spark config to force default credentials chain for AWS (#674) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/values-eks.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 175dd2fcc4..178a1e1504 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -504,6 +504,10 @@ configmap: plugins: spark: spark-config-default: + - # We override the default credentials chain provider for Hadoop so that + # it can use the serviceAccount based IAM role or ec2 metadata based. + # This is more in line with how AWS works + - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - spark.kubernetes.allocation.batch.size: "50" - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" From 635fe7f8620fc255abb4f3d86ec396dd570d5589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Sat, 20 Mar 2021 00:01:15 +0100 Subject: [PATCH 10/92] Helm - Update contour GRPC annotation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize change: - 46864a53 Access GRPC services using projectcontour (#686) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/values-eks.yaml | 4 ++-- helm/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 178a1e1504..6e0a928da8 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -27,8 +27,8 @@ flyteadmin: configPath: /etc/flyte/config/*.yaml service: annotations: - contour.heptio.com/upstream-protocol.h2c: grpc - + projectcontour.io/upstream-protocol.h2c: grpc + affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/helm/values.yaml b/helm/values.yaml index 25583bce4f..60004ce957 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -66,7 +66,7 @@ datacatalog: # -- Service settings for Datacatalog service: annotations: - contour.heptio.com/upstream-protocol.h2c: grpc + projectcontour.io/upstream-protocol.h2c: grpc type: NodePort # -- Annotations for ServiceAccount attached to Datacatalog pods serviceAccountAnnotations: {} From fdd49e8c2cf2af0c51e9edd127198e9a9f59c3bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Sat, 20 Mar 2021 00:06:51 +0100 Subject: [PATCH 11/92] Helm - Enabling Event version 1 by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize change: - 658aecd7 Enabling Event version 1 by default (#689) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/values-eks.yaml | 1 + helm/values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 6e0a928da8..29f000b759 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -323,6 +323,7 @@ configmap: metadataStoragePrefix: - "metadata" - "admin" + eventVersion: 1 testing: host: http://flyteadmin scheduler: diff --git a/helm/values.yaml b/helm/values.yaml index 60004ce957..acd3eaca7d 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -443,6 +443,7 @@ configmap: metadataStoragePrefix: - "metadata" - "admin" + eventVersion: 1 testing: host: http://flyteadmin From ebb5e40f68af9e80383413b41dea7da3313bfd88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Sat, 20 Mar 2021 00:29:05 +0100 Subject: [PATCH 12/92] Helm - Improve getting started experience MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize change: - fbe7e684 Improve getting started experience (#694) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/values-eks.yaml | 11 ++++++----- helm/values.yaml | 9 +++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 29f000b759..2cdb3d29ab 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -378,13 +378,14 @@ configmap: task_resource_defaults: task_resources: defaults: - cpu: 100m + cpu: 1000m memory: 1000Mi - storage: 5Mi + storage: 1000Mi limits: - cpu: 15000m - memory: 100Gi - storage: 400Gi + cpu: 2 + memory: 8Gi + storage: 2000Mi + gpu: 1 admin: event: diff --git a/helm/values.yaml b/helm/values.yaml index acd3eaca7d..eb27e126ee 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -473,12 +473,13 @@ configmap: task_resources: defaults: cpu: 100m - memory: 1000Mi + memory: 100Mi storage: 5Mi limits: - cpu: 15000m - memory: 100Gi - storage: 400Gi + cpu: 2 + memory: 8Gi + storage: 20Mi + gpu: 1 # Admin configuration admin: From f7bcff27005bb092ad483594ca613cd872b2cb22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Sat, 20 Mar 2021 00:32:59 +0100 Subject: [PATCH 13/92] Helm - Exponential error back-off in case of system errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize change: - 0844a44b Exponential error back-off in case of system errors (#693) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/values.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index eb27e126ee..814f6e7b2e 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -542,9 +542,11 @@ configmap: batching-interval: 2s batch-size: -1 queue: - type: bucket - rate: 10 - capacity: 100 + type: maxof + rate: 100 + capacity: 1000 + base-delay: 5s + max-delay: 120s sub-queue: type: bucket rate: 10 From ddab0dfd0850fb20011abc56c1a3c768ea0168db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Sat, 20 Mar 2021 22:01:22 +0100 Subject: [PATCH 14/92] Helm - Trim sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize change: - cd4a3d5a Trim sandbox (#727) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/admin/deployment.yaml | 2 +- helm/templates/postgres/deployment.yaml | 3 +++ helm/templates/propeller/configmap.yaml | 2 ++ helm/values-sandbox.yaml | 24 ++++++++++++++++++++---- helm/values.yaml | 2 +- 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/helm/templates/admin/deployment.yaml b/helm/templates/admin/deployment.yaml index 075488534a..f5b6178a32 100644 --- a/helm/templates/admin/deployment.yaml +++ b/helm/templates/admin/deployment.yaml @@ -20,7 +20,7 @@ spec: initContainers: {{- if .Values.postgres.enabled }} - name: check-db-ready - image: postgres:10.1 + image: postgres:10.16-alpine command: - sh - -c diff --git a/helm/templates/postgres/deployment.yaml b/helm/templates/postgres/deployment.yaml index 13ec8209c3..ff678d36ad 100644 --- a/helm/templates/postgres/deployment.yaml +++ b/helm/templates/postgres/deployment.yaml @@ -20,6 +20,9 @@ spec: - image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}" imagePullPolicy: "{{ .Values.postgres.image.pullPolicy }}" name: postgres + env: + - name: POSTGRES_HOST_AUTH_METHOD + value: trust ports: - containerPort: 5432 name: postgres diff --git a/helm/templates/propeller/configmap.yaml b/helm/templates/propeller/configmap.yaml index c604b5656e..70b4d58a0e 100644 --- a/helm/templates/propeller/configmap.yaml +++ b/helm/templates/propeller/configmap.yaml @@ -35,9 +35,11 @@ data: {{- with .Values.configmap.resource_manager }} resource_manager.yaml: | {{ toYaml . | nindent 4 }} {{- end }} +{{- if .Values.sparkoperator.enabled }} {{- with .Values.configmap.spark }} spark.yaml: | {{ toYaml . | nindent 4 }} {{- end }} +{{- end }} {{- with .Values.configmap.storage }} storage.yaml: | {{ toYaml . | nindent 4 }} {{- end }} diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index 148d564ba3..f1fad19694 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -27,13 +27,17 @@ flyteconsole: {} # REDIS # -redis: {} +redis: + enabled: false # # POSTGRES # -postgres: {} +postgres: + image: + # Override postgres image to use alpine based (rather smaller) docker image + tag: "10.16-alpine" # # MINIO @@ -59,13 +63,15 @@ contour: # SPARKOPERATOR # -sparkoperator: {} +sparkoperator: + enabled: false # # PYTORCHOPERATOR # -pytorchoperator: {} +pytorchoperator: + enabled: false # # COMMON @@ -132,6 +138,16 @@ configmap: show-source: true level: 5 + # Disable Qubole + qubole: null + + # Use noop resource-manager + resource_manager: + propeller: + resourcemanager: + type: noop + redis: null + ab_project_resource_quota: | apiVersion: v1 kind: ResourceQuota diff --git a/helm/values.yaml b/helm/values.yaml index 814f6e7b2e..f48c07c97d 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -194,7 +194,7 @@ postgres: image: # -- Docker image for Postgres deployment repository: postgres - tag: "10.1" + tag: "10.16" pullPolicy: IfNotPresent # -- Default resources requests and limits for Postgres deployment resources: From bacb1321b4faba7f0e778192535275cc1c0596d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Sat, 20 Mar 2021 22:05:08 +0100 Subject: [PATCH 15/92] Helm - Update default RAM Request for tasks in sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize change: - 72c57053 Update default RAM Request for tasks in sandbox (#750) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/values-sandbox.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index f1fad19694..8db1cfd763 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -126,6 +126,7 @@ configmap: - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" - FLYTE_AWS_ACCESS_KEY_ID: minio - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage + default-memory: 200Mi task_logs: plugins: From 08a4581d3ff7590eb47ce3f4a2fa24a4425f6321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Sat, 20 Mar 2021 22:10:13 +0100 Subject: [PATCH 16/92] Helm - Update Admin task resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize change: - b13de214 Update Admin task resources (#753) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/values-sandbox.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index 8db1cfd763..ece51fe09a 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -107,6 +107,12 @@ configmap: endpoint: http://minio.flyte.svc.cluster.local:9000 region: us-east-1 + # Task default resources configuration + task_resource_defaults: + task_resources: + defaults: + memory: 200Mi + enabled_plugins: tasks: task-plugins: From ddbdaebf13c39624406a956b6d34af46e4a8f468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Sat, 20 Mar 2021 23:54:09 +0100 Subject: [PATCH 17/92] Helm - Add kubernetes dashboard and update deployments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on the following kustomize changes: - a14fd47f Update deployments with latest k8s library deps (#826) - f149af8e Add kubernetes dashboard to sandbox overlay (#789) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- .gitignore | 1 + helm/Chart.yaml | 5 +++-- helm/templates/admin/cronjob.yaml | 1 - helm/templates/admin/deployment.yaml | 4 ---- helm/templates/datacatalog/deployment.yaml | 2 -- helm/values-sandbox.yaml | 15 ++++++++++++++- helm/values.yaml | 17 +++++++++-------- 7 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index bdde967a1d..62bdea5a24 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ docs/flytekit/flytekit.interfaces.html docs/searchindex.js docs/ __pycache__/ +/helm/charts/ \ No newline at end of file diff --git a/helm/Chart.yaml b/helm/Chart.yaml index b6bb75125a..efbd9675f3 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -14,5 +14,6 @@ dependencies: version: 1.0.6 repository: https://googlecloudplatform.github.io/spark-on-k8s-operator condition: spark.enabled - - + - name: kubernetes-dashboard + version: 4.0.2 + repository: https://kubernetes.github.io/dashboard/ diff --git a/helm/templates/admin/cronjob.yaml b/helm/templates/admin/cronjob.yaml index 68477543ed..28fe50d395 100644 --- a/helm/templates/admin/cronjob.yaml +++ b/helm/templates/admin/cronjob.yaml @@ -17,7 +17,6 @@ spec: containers: - command: - flyteadmin - - --logtostderr - --config - {{ .Values.flyteadmin.configPath }} - clusterresource diff --git a/helm/templates/admin/deployment.yaml b/helm/templates/admin/deployment.yaml index f5b6178a32..1f7003c88d 100644 --- a/helm/templates/admin/deployment.yaml +++ b/helm/templates/admin/deployment.yaml @@ -28,7 +28,6 @@ spec: {{- end }} - command: - flyteadmin - - --logtostderr - --config - {{ .Values.flyteadmin.configPath }} - migrate @@ -41,7 +40,6 @@ spec: name: config-volume - command: - flyteadmin - - --logtostderr - --config - {{ .Values.flyteadmin.configPath }} - migrate @@ -57,7 +55,6 @@ spec: name: config-volume - command: - flyteadmin - - --logtostderr - --config - {{ .Values.flyteadmin.configPath }} - clusterresource @@ -73,7 +70,6 @@ spec: containers: - command: - flyteadmin - - --logtostderr - --config - {{ .Values.flyteadmin.configPath }} {{- with .Values.flyteadmin.extraArgs }} diff --git a/helm/templates/datacatalog/deployment.yaml b/helm/templates/datacatalog/deployment.yaml index 23981b1c0a..1e3e2b6c6a 100644 --- a/helm/templates/datacatalog/deployment.yaml +++ b/helm/templates/datacatalog/deployment.yaml @@ -20,7 +20,6 @@ spec: initContainers: - command: - datacatalog - - --logtostderr - --config - {{ .Values.datacatalog.configPath }} - migrate @@ -34,7 +33,6 @@ spec: containers: - command: - datacatalog - - --logtostderr - --config - {{ .Values.datacatalog.configPath }} {{- with .Values.datacatalog.extraArgs }} diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index ece51fe09a..b4324a194f 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -73,6 +73,19 @@ sparkoperator: pytorchoperator: enabled: false +# +# KUBERNETES DASHBOARD +# +kubernetes-dashboard: + extraArgs: + - --enable-skip-login + - --enable-insecure-login + - --disable-settings-authorizer + protocolHttp: true + service: + type: NodePort + externalPort: 30082 + # # COMMON # @@ -138,7 +151,7 @@ configmap: plugins: logs: kubernetes-enabled: true - kubernetes-url: http://localhost:30082 + kubernetes-template-uri: "http://localhost:30082/#/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace }}" logger: logger: diff --git a/helm/values.yaml b/helm/values.yaml index f48c07c97d..02bce6577e 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -8,8 +8,8 @@ flyteadmin: replicaCount: 1 image: # -- Docker image for Flyteadmin deployment - repository: ghcr.io/lyft/flyteadmin - tag: v0.3.29 + repository: ghcr.io/flyteorg/flyteadmin + tag: v0.3.38 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteadmin deployment resources: @@ -48,8 +48,8 @@ datacatalog: replicaCount: 1 image: # -- Docker image for Datacatalog deployment - repository: ghcr.io/lyft/datacatalog - tag: v0.2.130 + repository: ghcr.io/flyteorg/datacatalog + tag: v0.3.0 pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment resources: @@ -89,7 +89,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: ghcr.io/flyteorg/flytepropeller - tag: v0.5.13 + tag: v0.7.1 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -123,8 +123,8 @@ flyteconsole: replicaCount: 1 image: # -- Docker image for Flyteconsole deployment - repository: ghcr.io/lyft/flyteconsole - tag: v0.19.1 + repository: ghcr.io/flyteorg/flyteconsole + tag: v0.19.6 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment resources: @@ -361,7 +361,8 @@ common: ingress: # --- enable or disable creating Ingress for Flyte. Relevant to disable when using e.g. Istio as ingress controller. enabled: true - annotations: {} + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" flyteNamespaceTemplate: # --- enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. enabled: false From cfcd2636d9090abe5566fd2f59c514103febacab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Wed, 31 Mar 2021 14:16:45 +0200 Subject: [PATCH 18/92] Helm - Update EKS Ingress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use default Docker images in EKS Helm values. Based on the following kustomize changes: - 4e0eed69 Update EKS Ingress and Instructions (#825) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/Chart.yaml | 1 + helm/templates/common/ingress.yaml | 285 +++++++++++++++++++---------- helm/values-eks.yaml | 48 ++--- helm/values-gcp.yaml | 9 + helm/values-sandbox.yaml | 6 +- helm/values.yaml | 22 ++- 6 files changed, 239 insertions(+), 132 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index efbd9675f3..b66ba74db3 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -17,3 +17,4 @@ dependencies: - name: kubernetes-dashboard version: 4.0.2 repository: https://kubernetes.github.io/dashboard/ + condition: kubernetes-dashboard.enabled diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index 6bbf9f10e7..934f360e80 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -1,114 +1,207 @@ {{- if .Values.common.ingress.enabled }} - -{{- if .Values.contour.enabled }} ---- -apiVersion: extensions/v1beta1 +# HTTP Routes +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ template "flyte.name" . }} namespace: {{ template "flyte.namespace" . }} + annotations: {{- with .Values.common.ingress.annotations }} - annotations: {{ toYaml . | nindent 4}} + {{ toYaml . | nindent 4}} + {{- end }} + {{- with .Values.common.ingress.httpAnnotations }} + {{ toYaml . | nindent 4}} {{- end }} spec: rules: - - http: - paths: - - backend: - serviceName: flyteconsole - servicePort: 80 - path: /console - - backend: - serviceName: flyteconsole - servicePort: 80 - path: /__webpack_hmr - - backend: - serviceName: flyteadmin - servicePort: 80 - path: /api - - backend: - serviceName: flyteadmin - servicePort: 80 - path: /healthcheck - - backend: - serviceName: flyteadmin - servicePort: 80 - path: /v1 - - backend: - serviceName: flyteadmin - servicePort: 81 - path: /flyteidl.service.AdminService - - backend: - serviceName: flyteadmin - servicePort: 87 - path: /openapi - -{{- else }} ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: {{ template "flyte.name" . }}-http1 - namespace: {{ template "flyte.namespace" . }} - annotations: - alb.ingress.kubernetes.io/backend-protocol-version: HTTP1 - {{- with .Values.common.ingress.annotations }} - {{- toYaml . | nindent 4}} - {{- end }} -spec: - rules: - - {{- with .Values.common.ingress.host }} - host: {{ . }} - {{- end }} - http: - paths: - - backend: - serviceName: flyteconsole - servicePort: 80 - path: /console* - - backend: - serviceName: flyteconsole - servicePort: 80 - path: /__webpack_hmr* - - backend: - serviceName: flyteadmin - servicePort: 80 - path: /api* - - backend: - serviceName: flyteadmin - servicePort: 80 - path: /healthcheck* - - backend: - serviceName: flyteadmin - servicePort: 80 - path: /v1* - - backend: - serviceName: flyteadmin - servicePort: 87 - path: /openapi* + - http: + paths: + {{- if eq .Values.common.ingress.controllerType "ALB" }} + - path: /* + pathType: ImplementationSpecific + backend: + service: + name: ssl-redirect + port: + name: use-annotation + {{- end }} + # This is useful only for frontend development + {{- if .Values.common.ingress.webpackHMR }} + - path: /__webpack_hmr + pathType: ImplementationSpecific + backend: + service: + name: flyteconsole + port: + number: 80 + {{- end }} + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 87 + # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml + - path: /console + pathType: ImplementationSpecific + backend: + service: + name: flyteconsole + port: + number: 80 + - path: /console/* + pathType: ImplementationSpecific + backend: + service: + name: flyteconsole + port: + number: 80 + - path: /api + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /api/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /healthcheck + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /v1/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /.well-known/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /login + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /login/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /logout + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /logout/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /callback + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /callback/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /me + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /config + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /config/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + {{- with .Values.common.ingress.host }} + host: {{ . }} + {{- end }} --- -apiVersion: extensions/v1beta1 +# GRPC Routes +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ template "flyte.name" . }}-grpc namespace: {{ template "flyte.namespace" . }} annotations: - alb.ingress.kubernetes.io/backend-protocol-version: GRPC - {{- with .Values.common.ingress.annotations }} - {{- toYaml . | nindent 4}} - {{- end }} + {{- with .Values.common.ingress.annotations }} + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.common.ingress.grpcAnnotations }} + {{- toYaml . | nindent 4}} + {{- end }} spec: rules: - - {{- with .Values.common.ingress.host }} - host: {{ . }} - {{- end }} - http: - paths: - - backend: - serviceName: flyteadmin - servicePort: 81 - path: /flyteidl.service.AdminService* - -{{- end }} - + - http: + paths: + # NOTE: Port 81 in flyteadmin is the GRPC server port for + # FlyteAdmin. + - path: /flyteidl.service.AdminService + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 81 + - path: /flyteidl.service.AdminService/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 81 + {{- with .Values.common.ingress.host }} + host: {{ . }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 2cdb3d29ab..16f7399bb0 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -5,10 +5,6 @@ flyteadmin: replicaCount: 2 - image: - repository: docker.io/lyft/flyteadmin - tag: v0.3.7 - pullPolicy: IfNotPresent # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html serviceAccountAnnotations: @@ -43,10 +39,6 @@ flyteadmin: datacatalog: replicaCount: 2 - image: - repository: docker.io/lyft/datacatalog - tag: v0.2.2 - pullPolicy: IfNotPresent # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html serviceAccountAnnotations: @@ -82,10 +74,6 @@ datacatalog: flytepropeller: replicaCount: 2 - image: - repository: docker.io/lyft/flytepropeller - tag: v0.4.2 - pullPolicy: IfNotPresent # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html serviceAccountAnnotations: @@ -115,10 +103,6 @@ flytepropeller: flyteconsole: replicaCount: 2 - image: - repository: docker.io/lyft/flyteconsole - tag: v0.12.1 - pullPolicy: IfNotPresent resources: limits: cpu: 250m @@ -141,10 +125,6 @@ flyteconsole: redis: enabled: true replicaCount: 1 - image: - repository: docker.io/bitnami/redis - tag: 4.0.2-r1 - pullPolicy: IfNotPresent resources: requests: cpu: 100m @@ -219,11 +199,21 @@ common: annotations: # aws-load-balancer-controller v2.1 or higher is required - https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.1/ kubernetes.io/ingress.class: alb - alb.ingress.kubernetes.io/group.name: flyte-ingress - alb.ingress.kubernetes.io/scheme: internal - alb.ingress.kubernetes.io/target-type: ip - alb.ingress.kubernetes.io/backend-protocol: HTTP - alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS": 443}]' + alb.ingress.kubernetes.io/tags: service_instance=production + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + # Instruct ALB Controller to not create multiple load balancers (and hence maintain a single endpoint for both GRPC and Http) + alb.ingress.kubernetes.io/group.name: flytesystem + # Replace certificate Arn with one deployed to your EKS cluster. Follow instructions in README.md + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 + # --- Extra Ingress annotations applied only to the HTTP ingress. + httpAnnotations: + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + # --- Extra Ingress annotations applied only to the GRPC ingress. + grpcAnnotations: + alb.ingress.kubernetes.io/backend-protocol-version: GRPC + alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]' + controllerType: "ALB" databaseSecret: name: db-pass secretManifest: @@ -410,14 +400,6 @@ configmap: writer: maxItems: 10000 - copilot: - plugins: - k8s: - co-pilot: - name: flyte-copilot- - image: docker.io/lyft/flytecopilot:v0.3.35 - start-timeout: 30s - core: propeller: rawoutput-prefix: s3://s3-bucket-for-flyte/ diff --git a/helm/values-gcp.yaml b/helm/values-gcp.yaml index 31d305c7a1..330d59200e 100644 --- a/helm/values-gcp.yaml +++ b/helm/values-gcp.yaml @@ -1,3 +1,12 @@ +# +# COMMON +# + +common: + databaseSecret: {} + ingress: + controllerType: "GKE" + # # CONFIGMAPS # diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index b4324a194f..f9ba146d78 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -77,6 +77,7 @@ pytorchoperator: # KUBERNETES DASHBOARD # kubernetes-dashboard: + enabled: true extraArgs: - --enable-skip-login - --enable-insecure-login @@ -92,7 +93,10 @@ kubernetes-dashboard: common: databaseSecret: {} - ingress: {} + ingress: + # enable HMR route to flyteconsole for frontend development. + webpackHMR: true + controllerType: "contour" flyteNamespaceTemplate: {} # diff --git a/helm/values.yaml b/helm/values.yaml index 02bce6577e..c9eaa8dda1 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -348,6 +348,13 @@ pytorchoperator: # -- affinity for Pytorchoperator deployment affinity: {} +# +# KUBERNETES DASHBOARD +# + +kubernetes-dashboard: + enabled: false + # # COMMON SETTINGS # @@ -359,12 +366,23 @@ common: # -- Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets secretManifest: {} ingress: - # --- enable or disable creating Ingress for Flyte. Relevant to disable when using e.g. Istio as ingress controller. + # --- Enable or disable creating Ingress for Flyte. Relevant to disable when using e.g. Istio as ingress controller. enabled: true + # --- Enable or disable HMR route to flyteconsole. This is useful only for frontend development. + webpackHMR: false + # --- Ingress annotations applied to both HTTP and GRPC ingresses. annotations: nginx.ingress.kubernetes.io/ssl-redirect: "false" + # --- Extra Ingress annotations applied only to the HTTP ingress. + httpAnnotations: {} + # --- Extra Ingress annotations applied only to the GRPC ingress. + grpcAnnotations: {} + # --- Ingress hostname + # host: + # --- controllerType defines the ingress controller type. Currently supported values are "contour", "ALB" or "GKE" + # controllerType: flyteNamespaceTemplate: - # --- enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. + # --- Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. enabled: false # From 31a657fab685ac33b5715b6af2e9d3fae182beef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Thu, 1 Apr 2021 21:10:16 +0200 Subject: [PATCH 19/92] Move GRPC annotations from ingress to services where possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/admin/service.yaml | 4 ++++ helm/templates/common/ingress.yaml | 35 +++++++++++++++++++++--------- helm/values-eks.yaml | 21 +++++++++--------- helm/values-gcp.yaml | 1 - helm/values-sandbox.yaml | 1 - helm/values.yaml | 23 +++++++++++--------- 6 files changed, 53 insertions(+), 32 deletions(-) diff --git a/helm/templates/admin/service.yaml b/helm/templates/admin/service.yaml index 2a2193940f..78b5acba81 100644 --- a/helm/templates/admin/service.yaml +++ b/helm/templates/admin/service.yaml @@ -11,6 +11,10 @@ spec: {{- with .Values.flyteadmin.service.type}} type: {{ . }} {{- end }} + {{- with .Values.flyteadmin.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{ . }} + {{- end }} ports: - name: redoc port: 87 diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index 934f360e80..c4288358fe 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -1,5 +1,4 @@ {{- if .Values.common.ingress.enabled }} -# HTTP Routes apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -9,14 +8,11 @@ metadata: {{- with .Values.common.ingress.annotations }} {{ toYaml . | nindent 4}} {{- end }} - {{- with .Values.common.ingress.httpAnnotations }} - {{ toYaml . | nindent 4}} - {{- end }} spec: rules: - http: paths: - {{- if eq .Values.common.ingress.controllerType "ALB" }} + {{- if .Values.common.ingress.albSSLRedirect }} - path: /* pathType: ImplementationSpecific backend: @@ -164,11 +160,30 @@ spec: name: flyteadmin port: number: 80 + {{- if not .Values.common.ingress.separateGrpcIngress }} + # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. + - path: /flyteidl.service.AdminService + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 81 + - path: /flyteidl.service.AdminService/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 81 + {{- end }} {{- with .Values.common.ingress.host }} host: {{ . }} {{- end }} +{{- if .Values.common.ingress.separateGrpcIngress }} +# Certain ingress controllers like nginx cannot serve HTTP 1 and GRPC with a single ingress because GRPC can only +# enabled on the ingress object, not on backend services (GRPC annotation is set on the ingress, not on the services). --- -# GRPC Routes apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -178,15 +193,14 @@ metadata: {{- with .Values.common.ingress.annotations }} {{- toYaml . | nindent 4}} {{- end }} - {{- with .Values.common.ingress.grpcAnnotations }} + {{- with .Values.common.ingress.separateGrpcIngressAnnotations }} {{- toYaml . | nindent 4}} {{- end }} spec: rules: - http: paths: - # NOTE: Port 81 in flyteadmin is the GRPC server port for - # FlyteAdmin. + # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. - path: /flyteidl.service.AdminService pathType: ImplementationSpecific backend: @@ -204,4 +218,5 @@ spec: {{- with .Values.common.ingress.host }} host: {{ . }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} +{{- end }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 16f7399bb0..63532c157b 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -23,7 +23,15 @@ flyteadmin: configPath: /etc/flyte/config/*.yaml service: annotations: - projectcontour.io/upstream-protocol.h2c: grpc + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" + # TODO add security groups + service.beta.kubernetes.io/aws-load-balancer-extra-security-groups: "sg-...,sg-...,sg-..." + # TODO alter domain + external-dns.alpha.kubernetes.io/hostname: "flyteadmin.subdomain.mydomain.com" + type: LoadBalancer + loadBalancerSourceRanges: + # TODO change source ip range if desired + - 0.0.0.0 affinity: podAntiAffinity: @@ -58,8 +66,8 @@ datacatalog: service: annotations: - contour.heptio.com/upstream-protocol.h2c: grpc - + cloud.google.com/load-balancer-type: Internal + type: LoadBalancer affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -206,14 +214,7 @@ common: alb.ingress.kubernetes.io/group.name: flytesystem # Replace certificate Arn with one deployed to your EKS cluster. Follow instructions in README.md alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 - # --- Extra Ingress annotations applied only to the HTTP ingress. - httpAnnotations: alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' - # --- Extra Ingress annotations applied only to the GRPC ingress. - grpcAnnotations: - alb.ingress.kubernetes.io/backend-protocol-version: GRPC - alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]' - controllerType: "ALB" databaseSecret: name: db-pass secretManifest: diff --git a/helm/values-gcp.yaml b/helm/values-gcp.yaml index 330d59200e..cbe96ceee7 100644 --- a/helm/values-gcp.yaml +++ b/helm/values-gcp.yaml @@ -5,7 +5,6 @@ common: databaseSecret: {} ingress: - controllerType: "GKE" # # CONFIGMAPS diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index f9ba146d78..750044b975 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -96,7 +96,6 @@ common: ingress: # enable HMR route to flyteconsole for frontend development. webpackHMR: true - controllerType: "contour" flyteNamespaceTemplate: {} # diff --git a/helm/values.yaml b/helm/values.yaml index c9eaa8dda1..558c17e8bb 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -9,7 +9,7 @@ flyteadmin: image: # -- Docker image for Flyteadmin deployment repository: ghcr.io/flyteorg/flyteadmin - tag: v0.3.38 + tag: v0.4.0 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteadmin deployment resources: @@ -27,7 +27,9 @@ flyteadmin: service: annotations: projectcontour.io/upstream-protocol.h2c: grpc + alb.ingress.kubernetes.io/backend-protocol-version: GRPC type: ClusterIP + loadBalancerSourceRanges: [] # -- Annotations for ServiceAccount attached to Flyteadmin pods serviceAccountAnnotations: {} # -- Annotations for Flyteadmin pods @@ -67,6 +69,7 @@ datacatalog: service: annotations: projectcontour.io/upstream-protocol.h2c: grpc + alb.ingress.kubernetes.io/backend-protocol-version: GRPC type: NodePort # -- Annotations for ServiceAccount attached to Datacatalog pods serviceAccountAnnotations: {} @@ -89,7 +92,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: ghcr.io/flyteorg/flytepropeller - tag: v0.7.1 + tag: v0.7.8 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -370,17 +373,17 @@ common: enabled: true # --- Enable or disable HMR route to flyteconsole. This is useful only for frontend development. webpackHMR: false + # --- separateGrpcIngress puts GRPC routes into a separate ingress if true. Required for certain ingress controllers like nginx. + separateGrpcIngress: false + # --- Extra Ingress annotations applied only to the GRPC ingress. Only makes sense if `separateGrpcIngress` is enabled. + separateGrpcIngressAnnotations: + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" # --- Ingress annotations applied to both HTTP and GRPC ingresses. - annotations: - nginx.ingress.kubernetes.io/ssl-redirect: "false" - # --- Extra Ingress annotations applied only to the HTTP ingress. - httpAnnotations: {} - # --- Extra Ingress annotations applied only to the GRPC ingress. - grpcAnnotations: {} + annotations: {} + # --- albSSLRedirect adds a special route for ssl redirect. Only useful in combination with the AWS LoadBalancer Controller. + albSSLRedirect: false # --- Ingress hostname # host: - # --- controllerType defines the ingress controller type. Currently supported values are "contour", "ALB" or "GKE" - # controllerType: flyteNamespaceTemplate: # --- Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. enabled: false From c30aad9af8719d206f8939f9422caf723b0cf95a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Thu, 1 Apr 2021 21:37:45 +0200 Subject: [PATCH 20/92] Improve installation docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/README.MD | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/helm/README.MD b/helm/README.MD index 227b37c1a7..60a0acbcb1 100644 --- a/helm/README.MD +++ b/helm/README.MD @@ -5,12 +5,37 @@ A Helm chart for Flyte ### SANDBOX INSTALLATION: -- Install `helm 3` release version -- Generate `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` +- [Install helm 3](https://helm.sh/docs/intro/install/) +- Fetch chart dependencies `helm dep up` +- Install Flyte sandbox: + +```bash +cd helm +helm install -n flyte -f values-sandbox.yaml --create-namespace flyte . +``` + +Customize your installation by changing settings in `values-sandbox.yaml`. +You can use the helm diff plugin to review any value changes you've made to your values: + +```bash +helm plugin install https://github.com/databus23/helm-diff +helm diff upgrade -f values-sandbox.yaml flyte . +``` + +Then apply your changes: +```bash +helm upgrade -f values-sandbox.yaml flyte . +``` + +#### Alternative: Generate raw kubernetes yaml with helm template +- `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` - Deploy the manifest `kubectl apply -f flyte_generated_sandbox.yaml` + + - When all pods are running - run end2end tests: `kubectl apply -f ../end2end/tests/endtoend.yaml` - Get flyte host `minikube service contour -n heptio-contour --url`. And then visit `http:///console` + ### CONFIGURATION NOTES: - The docker images, their tags and other default parameters are configured in `values.yaml` file. - Each Flyte installation type should have separate `values-*.yaml` file: for sandbox, EKS and etc. The configuration in `values.yaml` and the choosen config `values-*.yaml` are merged when generating the deployment manifest. From a143dcbc6543226d704590def02c62ba072a00e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Mon, 12 Apr 2021 12:53:16 +0200 Subject: [PATCH 21/92] Downgrade Ingress API version for K8S 1.18 compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/common/ingress.yaml | 151 ++++++++++------------------- 1 file changed, 51 insertions(+), 100 deletions(-) diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index c4288358fe..e64cca6e5c 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -1,12 +1,11 @@ {{- if .Values.common.ingress.enabled }} -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ template "flyte.name" . }} namespace: {{ template "flyte.namespace" . }} - annotations: {{- with .Values.common.ingress.annotations }} - {{ toYaml . | nindent 4}} + annotations: {{ toYaml . | nindent 4}} {{- end }} spec: rules: @@ -16,166 +15,122 @@ spec: - path: /* pathType: ImplementationSpecific backend: - service: - name: ssl-redirect - port: - name: use-annotation + serviceName: ssl-redirect + servicePort: use-annotation {{- end }} # This is useful only for frontend development {{- if .Values.common.ingress.webpackHMR }} - path: /__webpack_hmr pathType: ImplementationSpecific backend: - service: - name: flyteconsole - port: - number: 80 + serviceName: flyteconsole + servicePort: 80 {{- end }} # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 87 + serviceName: flyteadmin + servicePort: 87 # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml - path: /console pathType: ImplementationSpecific backend: - service: - name: flyteconsole - port: - number: 80 + serviceName: flyteconsole + servicePort: 80 - path: /console/* pathType: ImplementationSpecific backend: - service: - name: flyteconsole - port: - number: 80 + serviceName: flyteconsole + servicePort: 80 - path: /api pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /api/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /healthcheck pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /v1/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /.well-known/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /login pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /login/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /logout pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /logout/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /callback pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /callback/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /me pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /config pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /config/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 {{- if not .Values.common.ingress.separateGrpcIngress }} # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. - path: /flyteidl.service.AdminService pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 81 + serviceName: flyteadmin + servicePort: 81 - path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 81 + serviceName: flyteadmin + servicePort: 81 {{- end }} {{- with .Values.common.ingress.host }} host: {{ . }} @@ -184,7 +139,7 @@ spec: # Certain ingress controllers like nginx cannot serve HTTP 1 and GRPC with a single ingress because GRPC can only # enabled on the ingress object, not on backend services (GRPC annotation is set on the ingress, not on the services). --- -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ template "flyte.name" . }}-grpc @@ -204,17 +159,13 @@ spec: - path: /flyteidl.service.AdminService pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 81 + serviceName: flyteadmin + servicePort: 81 - path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 81 + serviceName: flyteadmin + servicePort: 81 {{- with .Values.common.ingress.host }} host: {{ . }} {{- end }} From 682aa27e93ec4db6f3a4c35fa37c8fe5a8ec7818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Tue, 13 Apr 2021 09:58:47 +0200 Subject: [PATCH 22/92] Add tls option for ingress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/common/ingress.yaml | 12 ++++++++++++ helm/values-eks.yaml | 1 - helm/values-sandbox.yaml | 2 ++ helm/values.yaml | 2 ++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index e64cca6e5c..f5087bf9f3 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -135,6 +135,12 @@ spec: {{- with .Values.common.ingress.host }} host: {{ . }} {{- end }} + {{- if .Values.common.ingress.tls.enabled }} + tls: + - secretName: {{ .Release.Name }}-tls + hosts: + - {{ .Values.common.ingress.host }} + {{ end }} {{- if .Values.common.ingress.separateGrpcIngress }} # Certain ingress controllers like nginx cannot serve HTTP 1 and GRPC with a single ingress because GRPC can only # enabled on the ingress object, not on backend services (GRPC annotation is set on the ingress, not on the services). @@ -169,5 +175,11 @@ spec: {{- with .Values.common.ingress.host }} host: {{ . }} {{- end }} + {{- if .Values.common.ingress.tls.enabled }} + tls: + - secretName: {{ .Release.Name }}-tls + hosts: + - {{ .Values.common.ingress.host }} + {{ end }} {{- end }} {{- end }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 63532c157b..a74a897c52 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -20,7 +20,6 @@ flyteadmin: ephemeral-storage: 200Mi memory: 200Mi - configPath: /etc/flyte/config/*.yaml service: annotations: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index 750044b975..dff617bb7b 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -96,6 +96,8 @@ common: ingress: # enable HMR route to flyteconsole for frontend development. webpackHMR: true + tls: + enabled: false flyteNamespaceTemplate: {} # diff --git a/helm/values.yaml b/helm/values.yaml index 558c17e8bb..5f45bde061 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -384,6 +384,8 @@ common: albSSLRedirect: false # --- Ingress hostname # host: + tls: + enabled: true flyteNamespaceTemplate: # --- Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. enabled: false From b4158bea35a0d772ab0d77ee70a5c650faa69716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Wed, 14 Apr 2021 11:58:05 +0200 Subject: [PATCH 23/92] Remove redundant default values from EKS config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/values-eks.yaml | 220 +------------------------------------------ 1 file changed, 1 insertion(+), 219 deletions(-) diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index a74a897c52..3089b72968 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -5,16 +5,13 @@ flyteadmin: replicaCount: 2 - # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html serviceAccountAnnotations: eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte resources: limits: - cpu: 250m ephemeral-storage: 200Mi - memory: 500Mi requests: cpu: 50m ephemeral-storage: 200Mi @@ -46,23 +43,17 @@ flyteadmin: datacatalog: replicaCount: 2 - # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html serviceAccountAnnotations: eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte - resources: limits: cpu: 500m ephemeral-storage: 200Mi - memory: 500Mi requests: cpu: 50m ephemeral-storage: 200Mi memory: 200Mi - - configPath: /etc/datacatalog/config/*.yaml - service: annotations: cloud.google.com/load-balancer-type: Internal @@ -81,11 +72,9 @@ datacatalog: flytepropeller: replicaCount: 2 - # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html serviceAccountAnnotations: eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte - resources: limits: cpu: 500m @@ -95,7 +84,6 @@ flytepropeller: cpu: 50m ephemeral-storage: 200Mi memory: 200Mi - configPath: /etc/flyte/config*/*.yaml affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -113,10 +101,6 @@ flyteconsole: resources: limits: cpu: 250m - memory: 250Mi - requests: - cpu: 10m - memory: 50Mi affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -130,8 +114,6 @@ flyteconsole: # redis: - enabled: true - replicaCount: 1 resources: requests: cpu: 100m @@ -164,11 +146,6 @@ contour: sparkoperator: enabled: true - replicaCount: 1 - image: - repository: gcr.io/spark-operator/spark-operator - tag: v2.4.0-v1beta1-0.9.0 - pullPolicy: IfNotPresent resources: limits: cpu: 1000m @@ -182,12 +159,6 @@ sparkoperator: # pytorchoperator: - enabled: true - replicaCount: 1 - image: - repository: gcr.io/kubeflow-images-public/pytorch-operator - tag: v1.0.0-g047cf0f - pullPolicy: IfNotPresent resources: limits: cpu: 1000m @@ -243,32 +214,6 @@ common: # configmap: - - console: - BASE_URL: /console - CONFIG_DIR: /etc/flyte/config - - cluster_resources: - cluster_resources: - templatePath: "/etc/flyte/clusterresource/templates" - customData: - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" - refresh: 5m - db: database: port: 5432 @@ -276,23 +221,10 @@ configmap: host: dbname: flyte-development passwordPath: /etc/db/pass.txt - - domain: - domains: - - id: development - name: development - - id: staging - name: staging - - id: production - name: production - remote_data: remoteData: region: us-east-1 scheme: aws - signedUrls: - durationMinutes: 3 - server: server: httpPort: 8088 @@ -306,16 +238,6 @@ configmap: - "*" allowedHeaders: - "Content-Type" - flyteadmin: - roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin scheduler: eventScheduler: scheme: aws @@ -349,13 +271,7 @@ configmap: spark: - project: flytetester - project: spark-workflows - - - datacatalogServer: - datacatalog: - storage-prefix: metadata/datacatalog - metrics-scope: datacatalog - profiler-port: 10254 + storage: storage: @@ -372,69 +288,19 @@ configmap: memory: 1000Mi storage: 1000Mi limits: - cpu: 2 - memory: 8Gi storage: 2000Mi - gpu: 1 - - admin: - event: - type: admin - rate: 500 - capacity: 1000 - admin: - endpoint: flyteadmin:81 - insecure: true - - catalog: - catalog-cache: - endpoint: datacatalog:89 - type: datacatalog - insecure: true - - catalog_cache: - plugins: - catalogCache: - reader: - maxItems: 10000 - writer: - maxItems: 10000 core: propeller: rawoutput-prefix: s3://s3-bucket-for-flyte/ - metadata-prefix: metadata/propeller workers: 40 gc-interval: 12h max-workflow-retries: 50 - workflow-reeval-duration: 30s - downstream-eval-duration: 30s - limit-namespace: "all" - prof-port: 10254 - metrics-prefix: flyte - enable-admin-launcher: true - leader-election: - lock-config-map: - name: propeller-leader - namespace: flyte - enabled: true - lease-duration: 15s - renew-deadline: 10s - retry-period: 2s kube-client-config: qps: 100 burst: 25 timeout: 30s queue: - type: batch - batching-interval: 2s - batch-size: -1 - queue: - type: maxof - rate: 100 - capacity: 1000 - base-delay: 5s - max-delay: 120s sub-queue: type: bucket rate: 100 @@ -444,7 +310,6 @@ configmap: enabled_plugins: tasks: - max-plugin-phase-versions: 1000000 task-plugins: enabled-plugins: - container @@ -459,31 +324,10 @@ configmap: container_array: k8s-array pytorch: pytorch - k8s: - plugins: - k8s: - default-cpus: 100m - default-memory: 100Mi - logger: logger: - show-source: true level: 5 - qubole: - plugins: - qubole: - quboleTokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN" - - resource_manager: - propeller: - resourcemanager: - type: redis - resourceMaxQuota: 10000 - redis: - hostPath: redis-resource-manager:6379 - hostKey: mypassword - spark: plugins: spark: @@ -506,20 +350,6 @@ configmap: - spark.blacklist.timeout: "5m" - spark.task.maxfailures: "8" - task_logs: - plugins: - logs: - kubernetes-enabled: false - - aa_namespace: | - apiVersion: v1 - kind: Namespace - metadata: - name: {{ namespace }} - spec: - finalizers: - - kubernetes - ab_project_resource_quota: | apiVersion: v1 kind: ResourceQuota @@ -546,51 +376,3 @@ configmap: type: s3 container: my-s3-bucket enable-multicontainer: true - - ad_spark_role: | - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: Role - metadata: - name: spark-role - namespace: {{ namespace }} - rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - '*' - - apiGroups: - - "" - resources: - - services - verbs: - - '*' - - apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' - - ae_spark_service_account: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: spark - namespace: {{ namespace }} - - af_spark_role_binding: | - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: RoleBinding - metadata: - name: spark-role-binding - namespace: {{ namespace }} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: spark-role - subjects: - - kind: ServiceAccount - name: spark - namespace: {{ namespace }} From 4c8c7a5826df8106c42c61054c6d1bb87b6b627c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Wed, 14 Apr 2021 16:36:22 +0200 Subject: [PATCH 24/92] Simplify Helm storage configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/_helpers.tpl | 36 +++++++++++++++++++++++ helm/templates/admin/configmap.yaml | 4 +-- helm/templates/datacatalog/configmap.yaml | 5 ++-- helm/templates/propeller/configmap.yaml | 8 +++-- helm/values-eks.yaml | 9 ------ helm/values-sandbox.yaml | 12 -------- helm/values.yaml | 29 ++++++++++++------ 7 files changed, 64 insertions(+), 39 deletions(-) diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 244619bbad..3ae15cccf4 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -159,3 +159,39 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} name: {{ . }} {{- end }} {{- end }} + +{{- define "storage" -}} +storage: +{{- if eq .Values.storage.type "s3" }} + type: s3 + container: {{ .Values.storage.bucketName | quote }} + connection: + auth-type: iam + region: {{ .Values.storage.s3.region }} +{{- else if eq .Values.storage.type "gcs" }} + type: stow + stow: + kind: google + config: + json: "" + project_id: {{ .Values.storage.gcs.projectId }} + scopes: https://www.googleapis.com/auth/devstorage.read_write + container: {{ .Values.storage.bucketName | quote }} +{{- else if eq .Values.storage.type "sandbox" }} + type: minio + container: {{ .Values.storage.bucketName | quote }} + connection: + access-key: minio + auth-type: accesskey + secret-key: miniostorage + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 +{{- else if eq .Values.storage.type "custom" }} +{{- with .Values.storage.custom -}} + {{ toYaml . | nindent 2 }} +{{- end }} +{{- end }} + limits: + maxDownloadMBs: 10 +{{- end }} \ No newline at end of file diff --git a/helm/templates/admin/configmap.yaml b/helm/templates/admin/configmap.yaml index a061058caa..d344fe9172 100644 --- a/helm/templates/admin/configmap.yaml +++ b/helm/templates/admin/configmap.yaml @@ -23,9 +23,7 @@ data: {{- with .Values.configmap.server }} server.yaml: | {{ toYaml . | nindent 4 }} {{- end }} -{{- with .Values.configmap.storage }} - storage.yaml: | {{ toYaml . | nindent 4 }} -{{- end }} + storage.yaml: | {{ include "storage" . | nindent 4 }} {{- with .Values.configmap.task_resource_defaults }} task_resource_defaults.yaml: | {{ toYaml . | nindent 4 }} {{- end }} diff --git a/helm/templates/datacatalog/configmap.yaml b/helm/templates/datacatalog/configmap.yaml index dce73e47e4..0e35f32cef 100644 --- a/helm/templates/datacatalog/configmap.yaml +++ b/helm/templates/datacatalog/configmap.yaml @@ -14,6 +14,5 @@ data: {{- with .Values.configmap.datacatalogServer }} server.yaml: | {{ toYaml . | nindent 4 }} {{- end }} -{{- with .Values.configmap.storage }} - storage.yaml: | {{ toYaml . | nindent 4 }} -{{- end }} + storage.yaml: | {{ include "storage" . | nindent 4 }} + diff --git a/helm/templates/propeller/configmap.yaml b/helm/templates/propeller/configmap.yaml index 70b4d58a0e..593d968248 100644 --- a/helm/templates/propeller/configmap.yaml +++ b/helm/templates/propeller/configmap.yaml @@ -40,9 +40,11 @@ data: spark.yaml: | {{ toYaml . | nindent 4 }} {{- end }} {{- end }} -{{- with .Values.configmap.storage }} - storage.yaml: | {{ toYaml . | nindent 4 }} -{{- end }} + storage.yaml: | {{ include "storage" . | nindent 4 }} + cache.yaml: | + cache: + max_size_mbs: {{ .Values.flytepropeller.cacheSizeMbs }} + target_gc_percent: 70 {{- with .Values.configmap.task_logs }} task_logs.yaml: | {{ toYaml . | nindent 4 }} {{- end }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 3089b72968..77e49d0f22 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -272,15 +272,6 @@ configmap: - project: flytetester - project: spark-workflows - - storage: - storage: - type: s3 - container: s3-bucket-for-flyte - connection: - auth-type: iam - region: us-east-1 - task_resource_defaults: task_resources: defaults: diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index dff617bb7b..1965aea392 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -113,18 +113,6 @@ configmap: host: postgres dbname: flyte_development - storage: - storage: - type: minio - container: my-s3-bucket - connection: - access-key: minio - auth-type: accesskey - secret-key: miniostorage - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - # Task default resources configuration task_resource_defaults: task_resources: diff --git a/helm/values.yaml b/helm/values.yaml index 5f45bde061..b18ef46515 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -104,6 +104,7 @@ flytepropeller: cpu: 10m ephemeral-storage: 50Mi memory: 50Mi + cacheSizeMbs: 0 # -- Default regex string for searching configuration files configPath: /etc/flyte/config/*.yaml # -- Annotations for ServiceAccount attached to Flytepropeller pods @@ -390,6 +391,25 @@ common: # --- Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. enabled: false +# +# STORAGE SETTINGS +# + +storage: + # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. + type: sandbox + # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. + # bucketName: my-s3-bucket + # -- settings for storage type s3 + s3: + region: us-east-1 + # -- settings for storage type gcs + gcs: + # -- GCP project ID. Required for storage type gcs. + # projectId: + # -- Settings for storage type custom. See https://github:com/graymeta/stow for supported storage providers/settings. + custom: {} + # # CONFIGMAPS SETTINGS # @@ -482,15 +502,6 @@ configmap: httpPort: 8080 grpcServerReflection: true - # Storage configuration - storage: - storage: - type: minio - container: my-s3-bucket - connection: - auth-type: accesskey - region: us-east-1 - # Task default resources configuration task_resource_defaults: # -- Task default resources parameters From d2e93211a48456e420fb46c356c7c27d89f30638 Mon Sep 17 00:00:00 2001 From: Ketan Umare <16888709+kumare3@users.noreply.github.com> Date: Tue, 20 Apr 2021 08:25:51 -0700 Subject: [PATCH 25/92] [wip] Helm components (#921) Signed-off-by: Haytham Abuelfutuh --- .../admin/cluster_resource_configmap.yaml | 12 + helm/templates/admin/configmap.yaml | 23 +- helm/templates/admin/cronjob.yaml | 2 + helm/templates/admin/deployment.yaml | 4 + helm/templates/common/configmap.yaml | 27 -- helm/templates/datacatalog/configmap.yaml | 2 +- helm/values-eks.yaml | 325 +++++++++----- helm/values-sandbox.yaml | 186 +++++--- helm/values.yaml | 397 ++++++++++-------- 9 files changed, 583 insertions(+), 395 deletions(-) create mode 100644 helm/templates/admin/cluster_resource_configmap.yaml delete mode 100644 helm/templates/common/configmap.yaml diff --git a/helm/templates/admin/cluster_resource_configmap.yaml b/helm/templates/admin/cluster_resource_configmap.yaml new file mode 100644 index 0000000000..f3b7eeae1c --- /dev/null +++ b/helm/templates/admin/cluster_resource_configmap.yaml @@ -0,0 +1,12 @@ +{{- if .Values.cluster_resource_manager.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: clusterresource-template + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flyteadmin.labels" . | nindent 4 }} +data: +{{- range .Values.cluster_resource_manager.templates }} + {{ .key }}.yaml: | {{ .value | nindent 4 }} +{{- end }} +{{- end }} diff --git a/helm/templates/admin/configmap.yaml b/helm/templates/admin/configmap.yaml index d344fe9172..d1895000fc 100644 --- a/helm/templates/admin/configmap.yaml +++ b/helm/templates/admin/configmap.yaml @@ -5,10 +5,7 @@ metadata: namespace: {{ template "flyte.namespace" . }} labels: {{ include "flyteadmin.labels" . | nindent 4 }} data: -{{- with .Values.configmap.cluster_resources }} - cluster_resources.yaml: | {{ toYaml . | nindent 4 }} -{{- end }} -{{- with .Values.configmap.db }} +{{- with .Values.db }} db.yaml: | {{ toYaml . | nindent 4 }} {{- end }} {{- with .Values.configmap.domain }} @@ -17,9 +14,6 @@ data: {{- with .Values.configmap.logger }} logger.yaml: | {{ toYaml . | nindent 4 }} {{- end }} -{{- with .Values.configmap.remote_data }} - remote_data.yaml: | {{ toYaml . | nindent 4 }} -{{- end }} {{- with .Values.configmap.server }} server.yaml: | {{ toYaml . | nindent 4 }} {{- end }} @@ -27,3 +21,18 @@ data: {{- with .Values.configmap.task_resource_defaults }} task_resource_defaults.yaml: | {{ toYaml . | nindent 4 }} {{- end }} +{{- if .Values.workflow_notifications.enabled }} +{{- with .Values.workflow_notifications.config }} + notifications.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- end }} +{{- if .Values.cluster_resource_manager.enabled }} +{{- with .Values.cluster_resource_manager.config }} + cluster_resources.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- end }} +{{- if .Values.workflow_scheduler.enabled }} +{{- with .Values.workflow_scheduler.config }} + scheduler.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- end }} diff --git a/helm/templates/admin/cronjob.yaml b/helm/templates/admin/cronjob.yaml index 28fe50d395..0c13789fba 100644 --- a/helm/templates/admin/cronjob.yaml +++ b/helm/templates/admin/cronjob.yaml @@ -1,3 +1,4 @@ +{{- if .Values.cluster_resource_manager.enabled }} apiVersion: batch/v1beta1 kind: CronJob metadata: @@ -38,3 +39,4 @@ spec: - configMap: name: flyte-admin-config name: config-volume + {{- end }} \ No newline at end of file diff --git a/helm/templates/admin/deployment.yaml b/helm/templates/admin/deployment.yaml index 1f7003c88d..d088d44c6e 100644 --- a/helm/templates/admin/deployment.yaml +++ b/helm/templates/admin/deployment.yaml @@ -53,6 +53,7 @@ spec: volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} - mountPath: /etc/flyte/config name: config-volume + {{- if .Values.cluster_resource_manager.enabled }} - command: - flyteadmin - --config @@ -67,6 +68,7 @@ spec: name: resource-templates - mountPath: /etc/flyte/config name: config-volume + {{- end }} containers: - command: - flyteadmin @@ -115,9 +117,11 @@ spec: - configMap: name: flyte-admin-config name: config-volume + {{- if .Values.cluster_resource_manager.enabled }} - configMap: name: clusterresource-template name: resource-templates + {{- end }} {{- with .Values.flyteadmin.nodeSelector }} nodeSelector: {{ toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/common/configmap.yaml b/helm/templates/common/configmap.yaml deleted file mode 100644 index 4291ec676f..0000000000 --- a/helm/templates/common/configmap.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: clusterresource-template - namespace: {{ template "flyte.namespace" . }} - labels: {{ include "flyteadmin.labels" . | nindent 4 }} -data: -{{- with .Values.configmap.aa_namespace }} - aa_namespace.yaml: | {{ . | nindent 4 }} -{{- end }} -{{- with .Values.configmap.ab_project_resource_quota }} - ab_project-resource-quota.yaml: | {{ . | nindent 4 }} -{{- end }} -{{- with .Values.configmap.ac_project_copilot_dataconfig }} - ac_project-copilot-dataconfig.yaml: | {{ . | nindent 4 }} -{{- end }} -{{- if .Values.sparkoperator.enabled }} -{{- with .Values.configmap.ad_spark_role }} - ad_spark-role.yaml: | {{ . | nindent 4 }} -{{- end }} -{{- with .Values.configmap.ae_spark_service_account }} - ae_spark-service-account.yaml: | {{ . | nindent 4 }} -{{- end }} -{{- with .Values.configmap.af_spark_role_binding }} - af_spark-role-binding.yaml: | {{ . | nindent 4 }} -{{- end }} -{{- end }} diff --git a/helm/templates/datacatalog/configmap.yaml b/helm/templates/datacatalog/configmap.yaml index 0e35f32cef..2f05fe1c2d 100644 --- a/helm/templates/datacatalog/configmap.yaml +++ b/helm/templates/datacatalog/configmap.yaml @@ -5,7 +5,7 @@ metadata: namespace: {{ template "flyte.namespace" . }} labels: {{ include "datacatalog.labels" . | nindent 4 }} data: -{{- with .Values.configmap.db }} +{{- with .Values.db }} db.yaml: | {{ toYaml . | nindent 4 }} {{- end }} {{- with .Values.configmap.logger }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 77e49d0f22..6db61dd2ba 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -140,32 +140,7 @@ minio: contour: enabled: false -# -# SPARKOPERATOR -# -sparkoperator: - enabled: true - resources: - limits: - cpu: 1000m - memory: 1000Mi - requests: - cpu: 50m - memory: 250Mi - -# -# PYTORCHOPERATOR -# - -pytorchoperator: - resources: - limits: - cpu: 1000m - memory: 1000Mi - requests: - cpu: 50m - memory: 250Mi # # COMMON @@ -209,22 +184,36 @@ common: # key: k8s/data/path/to/secret # property: password + +# ----------------------------------------------------- +# Core dependencies that should be configured for Flyte to work on any platform +# Specifically 2 - Storage (s3, gcs etc), Production RDBMS - Aurora, CloudSQL etc +# ------------------------------------------------------ +# +# STORAGE SETTINGS +# + +storage: + # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. + type: s3 + # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. + bucketName: + s3: + region: us-east-1 + +db: + database: + port: 5432 + username: postgres + host: + dbname: flyte-development + passwordPath: /etc/db/pass.txt + # # CONFIGMAPS # configmap: - db: - database: - port: 5432 - username: postgres - host: - dbname: flyte-development - passwordPath: /etc/db/pass.txt - remote_data: - remoteData: - region: us-east-1 - scheme: aws server: server: httpPort: 8088 @@ -238,35 +227,7 @@ configmap: - "*" allowedHeaders: - "Content-Type" - scheduler: - eventScheduler: - scheme: aws - region: us-east-1 - scheduleRole: arn:aws:iam:::role/flyte_cron_scheduler_role - targetName: arn:aws:sqs:us-east-1::flyte-cron-scheduler-queue - scheduleNamePrefix: flyte - workflowExecutor: - scheme: aws - region: us-east-1 - scheduleQueueName: flyte-cron-scheduler-queue - accountId: - reconnectAttempts: 10 - reconnectDelaySeconds: 30 - notifications: - type: aws - region: us-east-1 - publisher: - topicName: arn:aws:sns:us-east-1::flyte-notifications-topic - processor: - queueName: flyte-notifications-queue - accountId: - emailer: - subject: "Flyte: {{ project }}/{{ domain }}/{{ launch_plan.name }} has '{{ phase }}'" - sender: "flyte@example.com" - body: | - Execution {{ workflow.project }}/{{ workflow.domain }}/{{ workflow.name }}/{{ name }} has {{ phase }}. - Details: https://flyte.example.com/console/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. - {{ error }} + task_type_whitelist: spark: - project: flytetester @@ -296,8 +257,6 @@ configmap: type: bucket rate: 100 capacity: 1000 - workflowStore: - policy: "ResourceVersionCache" enabled_plugins: tasks: @@ -319,51 +278,187 @@ configmap: logger: level: 5 - spark: - plugins: - spark: - spark-config-default: - - # We override the default credentials chain provider for Hadoop so that - # it can use the serviceAccount based IAM role or ec2 metadata based. - # This is more in line with how AWS works - - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" - - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - - spark.kubernetes.allocation.batch.size: "50" - - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" - - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.multipart.threshold: "536870912" - - spark.blacklist.enabled: "true" - - spark.blacklist.timeout: "5m" - - spark.task.maxfailures: "8" - - ab_project_resource_quota: | - apiVersion: v1 - kind: ResourceQuota - metadata: - name: project-quota - namespace: {{ namespace }} - spec: - hard: - limits.cpu: {{ projectQuotaCpu }} - limits.memory: {{ projectQuotaMemory }} - - ac_project_copilot_dataconfig: | - kind: ConfigMap - apiVersion: v1 - metadata: - name: flyte-data-config - namespace: {{ namespace }} - data: - config.yaml: | - storage: - connection: - auth-type: iam - region: us-east-1 - type: s3 - container: my-s3-bucket - enable-multicontainer: true + +# ---------------------------------------------------------------- +# Optional Modules +# Flyte built extensions that enable various additional features in Flyte. +# All these features are optional, but are critical to run certain features +# ------------------------------------------------------------------------ + +# -- +# Flyte uses a cloud hosted Cron scheduler to run workflows on a schedule. The following module is optional. Without, +# this module, you will not have scheduled launchplans/workflows. +workflow_scheduler: + enabled: true + config: + scheduler: + eventScheduler: + scheme: aws + region: us-east-1 + scheduleRole: arn:aws:iam:::role/flyte_cron_scheduler_role + targetName: arn:aws:sqs:us-east-1::flyte-cron-scheduler-queue + scheduleNamePrefix: flyte + workflowExecutor: + scheme: aws + region: us-east-1 + scheduleQueueName: flyte-cron-scheduler-queue + accountId: + reconnectAttempts: 10 + reconnectDelaySeconds: 30 + + +# -- +# Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of +# various events in their workflows +workflow_notifications: + enabled: true + config: + notifications: + type: aws + region: us-east-1 + publisher: + topicName: arn:aws:sns:us-east-1::flyte-notifications-topic + processor: + queueName: flyte-notifications-queue + accountId: + emailer: + subject: "Flyte: {{ project }}/{{ domain }}/{{ launch_plan.name }} has '{{ phase }}'" + sender: "flyte@example.com" + body: | + Execution {{ workflow.project }}/{{ workflow.domain }}/{{ workflow.name }}/{{ name }} has {{ phase }}. + Details: https://flyte.example.com/console/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. + {{ error }} + + +# -- Configuration for the Cluster resource manager component. This is an optional component, that enables automatic +# cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain +cluster_resource_manager: + # -- Enables the Cluster resource manager component + enabled: true + templates: + # -- Template for namespaces resources + - key: aa_namespace + value: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes + + - key: ab_project_resource_quota + value: | + apiVersion: v1 + kind: ResourceQuota + metadata: + name: project-quota + namespace: {{ namespace }} + spec: + hard: + limits.cpu: {{ projectQuotaCpu }} + limits.memory: {{ projectQuotaMemory }} + + # -- Template for spark role + - key: ac_project_copilot_dataconfig + value: | + kind: ConfigMap + apiVersion: v1 + metadata: + name: flyte-data-config + namespace: {{ namespace }} + data: + config.yaml: | + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + type: minio + container: my-s3-bucket + enable-multicontainer: true + + # -- Template for spark role + - key: ad_spark_role + value: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: Role + metadata: + name: spark-role + namespace: {{ namespace }} + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - '*' + - apiGroups: + - "" + resources: + - services + verbs: + - '*' + - apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' + + # -- Template for spark SA + - key: ae_spark_service_account + value: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: spark + namespace: {{ namespace }} + + # -- Template for spark role binding + - key: af_spark_role_binding + value: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: RoleBinding + metadata: + name: spark-role-binding + namespace: {{ namespace }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spark-role + subjects: + - kind: ServiceAccount + name: spark + namespace: {{ namespace }} + +# +# SPARKOPERATOR +# + +sparkoperator: + enabled: true + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 50m + memory: 250Mi + +# +# PYTORCHOPERATOR +# + +pytorchoperator: + enabled: true + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 50m + memory: 250Mi \ No newline at end of file diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index 1965aea392..c7304b037f 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -1,4 +1,11 @@ +# --------------------------------------------------------------------- +# Core System settings +# This section consists of Core components of Flyte and their deployment +# settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and +# Flyteconsole +# --------------------------------------------------------------------- + # # FLYTEADMIN # @@ -23,8 +30,15 @@ flytepropeller: {} flyteconsole: {} + +# ---------------------------------------------- +# Sandbox Configuration +# Sandbox allows to run flyte without any cloud dependencies and can be run even locally on your laptop. +# This is achieved by replacing cloud service dependencies by k8s local alternatives. These may not be ideal +# for a high performance setup, but are great to try out flyte +# ----------------------------------------------- # -# REDIS +# REDIS SETTINGS # redis: @@ -59,19 +73,6 @@ contour: nodePorts: http: 30081 -# -# SPARKOPERATOR -# - -sparkoperator: - enabled: false - -# -# PYTORCHOPERATOR -# - -pytorchoperator: - enabled: false # # KUBERNETES DASHBOARD @@ -87,8 +88,9 @@ kubernetes-dashboard: type: NodePort externalPort: 30082 +# ------------------------------------------------ # -# COMMON +# COMMON SETTINGS # common: @@ -100,18 +102,39 @@ common: enabled: false flyteNamespaceTemplate: {} + +# ----------------------------------------------------- +# Core dependencies that should be configured for Flyte to work on any platform +# Specifically 2 - Storage (s3, gcs etc), Production RDBMS - Aurora, CloudSQL etc +# ------------------------------------------------------ # -# CONFIGMAPS +# STORAGE SETTINGS # -configmap: +storage: + # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. + type: sandbox + # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. + bucketName: my-s3-bucket - db: - database: - port: 5432 - username: postgres - host: postgres - dbname: flyte_development +# Database configuration +db: + database: + port: 5432 + username: postgres + host: postgres + dbname: flyte_development + + + +# -------------------------------------------------------------------- +# Specializing your deployment using configuration +# ------------------------------------------------------------------- +# +# CONFIGMAPS SETTINGS +# + +configmap: # Task default resources configuration task_resource_defaults: @@ -119,6 +142,14 @@ configmap: defaults: memory: 200Mi + # Core propeller configuration + core: + propeller: + rawoutput-prefix: s3://my-s3-bucket/ + workers: 20 + max-workflow-retries: 30 + + # Plugins configuration enabled_plugins: tasks: task-plugins: @@ -131,6 +162,7 @@ configmap: sidecar: sidecar container_array: k8s-array + # Kubernetes plugin configuration k8s: plugins: k8s: @@ -139,55 +171,81 @@ configmap: - FLYTE_AWS_ACCESS_KEY_ID: minio - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage default-memory: 200Mi - - task_logs: - plugins: - logs: - kubernetes-enabled: true - kubernetes-template-uri: "http://localhost:30082/#/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace }}" + # Logger configuration logger: logger: show-source: true - level: 5 - - # Disable Qubole - qubole: null + level: 4 - # Use noop resource-manager + # Resource manager configuration resource_manager: propeller: resourcemanager: type: noop redis: null - ab_project_resource_quota: | - apiVersion: v1 - kind: ResourceQuota - metadata: - name: project-quota - namespace: {{ namespace }} - spec: - hard: - limits.cpu: {{ projectQuotaCpu }} - limits.memory: {{ projectQuotaMemory }} - - ac_project_copilot_dataconfig: | - kind: ConfigMap - apiVersion: v1 - metadata: - name: flyte-data-config - namespace: {{ namespace }} - data: - config.yaml: | - storage: - connection: - access-key: minio - auth-type: accesskey - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - secret-key: miniostorage - type: minio - container: my-s3-bucket - enable-multicontainer: true + # Tasks logs plugin configuration + task_logs: + plugins: + logs: + kubernetes-enabled: true + kubernetes-template-uri: "http://localhost:30082/#/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace }}" + +# ---------------------------------------------------------------- +# Optional Modules +# Flyte built extensions that enable various additional features in Flyte. +# All these features are optional, but are critical to run certain features +# ------------------------------------------------------------------------ + +# -- +# Flyte uses a cloud hosted Cron scheduler to run workflows on a schedule. The following module is optional. Without, +# this module, you will not have scheduled launchplans/workflows. +workflow_scheduler: + enabled: false + + +# -- +# Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of +# various events in their workflows +workflow_notifications: + enabled: false + + +# -- Configuration for the Cluster resource manager component. This is an optional component, that enables automatic +# cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain +cluster_resource_manager: + # -- Enables the Cluster resource manager component + enabled: true + + +# -------------------------------------------------------- +# Optional Plugins +# -------------------------------------------------------- + +# +# SPARKOPERATOR +# + +sparkoperator: + enabled: false + +# +# PYTORCHOPERATOR +# + +pytorchoperator: + enabled: false + +# ---------------------- +# Distributed tensorflow training using the TF-Operator +# + + +# ---------------------- +# Training on AWS Sagemaker using AWS Sagemaker operator +# + +# --------------------------- +# Flink jobs using the Flink Operator +# diff --git a/helm/values.yaml b/helm/values.yaml index b18ef46515..8bc0675b75 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,4 +1,11 @@ +# --------------------------------------------------------------------- +# Core System settings +# This section consists of Core components of Flyte and their deployment +# settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and +# Flyteconsole +# --------------------------------------------------------------------- + # # FLYTEADMIN SETTINGS # @@ -151,6 +158,13 @@ flyteconsole: # -- affinity for Flyteconsole deployment affinity: {} + +# ---------------------------------------------- +# Sandbox Configuration +# Sandbox allows to run flyte without any cloud dependencies and can be run even locally on your laptop. +# This is achieved by replacing cloud service dependencies by k8s local alternatives. These may not be ideal +# for a high performance setup, but are great to try out flyte +# ----------------------------------------------- # # REDIS SETTINGS # @@ -294,63 +308,6 @@ contour: # -- affinity for Contour deployment affinity: {} -# -# SPARKOPERATOR SETTINGS -# - -sparkoperator: - # --- enable or disable Sparkoperator deployment installation - enabled: true - # -- Replicas count for Sparkoperator deployment - replicaCount: 1 - image: - # -- Docker image for Sparkoperator - tag: v1beta2-1.2.0-3.0.0 # Set to v1beta2-1.1.2-2.4.5 for Spark 2 - # -- Default resources requests and limits for Sparkoperator - resources: - limits: - cpu: 1000m - memory: 500M - requests: - cpu: 10m - memory: 50M - -# -# PYTORCHOPERATOR SETTINGS -# - -pytorchoperator: - # --- enable or disable Pytorchoperator deployment installation - enabled: true # Set false to disable - # -- Replicas count for Pytorchoperator deployment - replicaCount: 1 - image: - # -- Docker image for Pytorchoperator - repository: gcr.io/kubeflow-images-public/pytorch-operator - tag: v1.0.0-g047cf0f - pullPolicy: IfNotPresent - # -- Default resources requests and limits for Pytorchoperator - resources: - limits: - cpu: 500m - memory: 1000M - requests: - cpu: 10m - memory: 50M - # -- Service settings for Pytorchoperator - service: - annotations: {} - type: ClusterIP - # -- Annotations for ServiceAccount attached to Pytorchoperator pods - serviceAccountAnnotations: {} - # -- Annotations for Pytorchoperator pods - podAnnotations: {} - # -- nodeSelector for Pytorchoperator deployment - nodeSelector: {} - # -- tolerations for Pytorchoperator deployment - tolerations: [] - # -- affinity for Pytorchoperator deployment - affinity: {} # # KUBERNETES DASHBOARD @@ -359,6 +316,7 @@ pytorchoperator: kubernetes-dashboard: enabled: false +# ------------------------------------------------ # # COMMON SETTINGS # @@ -391,6 +349,11 @@ common: # --- Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. enabled: false + +# ----------------------------------------------------- +# Core dependencies that should be configured for Flyte to work on any platform +# Specifically 2 - Storage (s3, gcs etc), Production RDBMS - Aurora, CloudSQL etc +# ------------------------------------------------------ # # STORAGE SETTINGS # @@ -399,7 +362,7 @@ storage: # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. type: sandbox # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. - # bucketName: my-s3-bucket + bucketName: my-s3-bucket # -- settings for storage type s3 s3: region: us-east-1 @@ -410,49 +373,30 @@ storage: # -- Settings for storage type custom. See https://github:com/graymeta/stow for supported storage providers/settings. custom: {} +# Database configuration +db: + database: {} + # port: 5432 + # username: postgres + # host: postgres + # dbname: flyte + # passwordPath: /etc/db/pass.txt + + + +# -------------------------------------------------------------------- +# Specializing your deployment using configuration +# ------------------------------------------------------------------- # # CONFIGMAPS SETTINGS # configmap: - # Configmap for Flyte console UI console: BASE_URL: /console CONFIG_DIR: /etc/flyte/config - # Configmap for ClusterResource parameters - cluster_resources: - # -- ClusterResource parameters - cluster_resources: - templatePath: "/etc/flyte/clusterresource/templates" - customData: - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" - refresh: 5m - - # Database configuration - db: - database: {} - # port: 5432 - # username: postgres - # host: postgres - # dbname: flyte - # passwordPath: /etc/db/pass.txt - # Domains configuration for Flyte projects domain: domains: @@ -463,14 +407,6 @@ configmap: - id: production name: production - # Remote data configuration - remote_data: - remoteData: - region: us-east-1 - scheme: local - signedUrls: - durationMinutes: 3 - # -- Flyteadmin server config server: server: @@ -533,15 +469,6 @@ configmap: type: datacatalog insecure: true - # Catalog cache configuration - catalog_cache: - plugins: - catalogCache: - reader: - maxItems: 10000 - writer: - maxItems: 10000 - # Copilot configuration copilot: plugins: @@ -590,7 +517,6 @@ configmap: # Plugins configuration enabled_plugins: tasks: - max-plugin-phase-versions: 1000000 task-plugins: enabled-plugins: [] @@ -610,12 +536,6 @@ configmap: show-source: true level: 4 - # Qubole plugin configuration - qubole: - plugins: - qubole: - quboleTokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN" - # Resource manager configuration resource_manager: # -- resource manager configuration @@ -627,8 +547,139 @@ configmap: hostPath: redis-resource-manager:6379 hostKey: mypassword + # Tasks logs plugin configuration + task_logs: + plugins: + logs: + kubernetes-enabled: false + +# ---------------------------------------------------------------- +# Optional Modules +# Flyte built extensions that enable various additional features in Flyte. +# All these features are optional, but are critical to run certain features +# ------------------------------------------------------------------------ + +# -- +# Flyte uses a cloud hosted Cron scheduler to run workflows on a schedule. The following module is optional. Without, +# this module, you will not have scheduled launchplans/workflows. +workflow_scheduler: + enabled: false + config: {} + + + +# -- +# Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of +# various events in their workflows +workflow_notifications: + enabled: false + config: {} + + + +# -- Configuration for the Cluster resource manager component. This is an optional component, that enables automatic +# cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain +cluster_resource_manager: + # -- Enables the Cluster resource manager component + enabled: true + # Configmap for ClusterResource parameters + config: + # -- ClusterResource parameters + cluster_resources: + templatePath: "/etc/flyte/clusterresource/templates" + customData: + production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "3000Mi" + refresh: 5m + + # -- Resource templates that should be applied + templates: + # -- Template for namespaces resources + - key: aa_namespace + value: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes + + - key: ab_project_resource_quota + value: | + apiVersion: v1 + kind: ResourceQuota + metadata: + name: project-quota + namespace: {{ namespace }} + spec: + hard: + limits.cpu: {{ projectQuotaCpu }} + limits.memory: {{ projectQuotaMemory }} + + # -- Template for spark role + - key: ac_project_copilot_dataconfig + value: | + kind: ConfigMap + apiVersion: v1 + metadata: + name: flyte-data-config + namespace: {{ namespace }} + data: + config.yaml: | + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + type: minio + container: my-s3-bucket + enable-multicontainer: true + + +# -------------------------------------------------------- +# Optional Plugins +# -------------------------------------------------------- + +# +# SPARK Plugin using the Spark Operator +# + +sparkoperator: + # --- enable or disable Sparkoperator deployment installation + enabled: true + # -- Replicas count for Sparkoperator deployment + replicaCount: 1 + image: + # -- Docker image for Sparkoperator + tag: v1beta2-1.2.0-3.0.0 # Set to v1beta2-1.1.2-2.4.5 for Spark 2 + # -- Default resources requests and limits for Sparkoperator + resources: + limits: + cpu: 1000m + memory: 500M + requests: + cpu: 10m + memory: 50M + # Spark plugin configuration - spark: + plugin_config: plugins: spark: # -- Spark default configuration @@ -647,69 +698,53 @@ configmap: - spark.blacklist.timeout: "5m" - spark.task.maxfailures: "8" - # Tasks logs plugin configuration - task_logs: - plugins: - logs: - kubernetes-enabled: false - # -- Template for namespaces resources - aa_namespace: | - apiVersion: v1 - kind: Namespace - metadata: - name: {{ namespace }} - spec: - finalizers: - - kubernetes - - # -- Template for spark role - ad_spark_role: | - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: Role - metadata: - name: spark-role - namespace: {{ namespace }} - rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - '*' - - apiGroups: - - "" - resources: - - services - verbs: - - '*' - - apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' - - # -- Template for spark SA - ae_spark_service_account: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: spark - namespace: {{ namespace }} - - # -- Template for spark role binding - af_spark_role_binding: | - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: RoleBinding - metadata: - name: spark-role-binding - namespace: {{ namespace }} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: spark-role - subjects: - - kind: ServiceAccount - name: spark - namespace: {{ namespace }} +# ---------------------- +# Distributed Pytorch execution using the K8s Pytorch operator +# + +pytorchoperator: + # --- enable or disable Pytorchoperator deployment installation + enabled: true # Set false to disable + # -- Replicas count for Pytorchoperator deployment + replicaCount: 1 + image: + # -- Docker image for Pytorchoperator + repository: gcr.io/kubeflow-images-public/pytorch-operator + tag: v1.0.0-g047cf0f + pullPolicy: IfNotPresent + # -- Default resources requests and limits for Pytorchoperator + resources: + limits: + cpu: 500m + memory: 1000M + requests: + cpu: 10m + memory: 50M + # -- Service settings for Pytorchoperator + service: + annotations: {} + type: ClusterIP + # -- Annotations for ServiceAccount attached to Pytorchoperator pods + serviceAccountAnnotations: {} + # -- Annotations for Pytorchoperator pods + podAnnotations: {} + # -- nodeSelector for Pytorchoperator deployment + nodeSelector: {} + # -- tolerations for Pytorchoperator deployment + tolerations: [] + # -- affinity for Pytorchoperator deployment + affinity: {} + +# ---------------------- +# Distributed tensorflow training using the TF-Operator +# + + +# ---------------------- +# Training on AWS Sagemaker using AWS Sagemaker operator +# + +# --------------------------- +# Flink jobs using the Flink Operator +# From 643a8b2fdb53cb03764004ef35df2660c13567d8 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 20 Apr 2021 08:57:25 -0700 Subject: [PATCH 26/92] Working helm deployment Signed-off-by: Haytham Abuelfutuh --- .gitignore | 3 +- helm/README.MD | 200 +++++++++++++------------------------------ helm/values-eks.yaml | 6 +- helm/values.yaml | 12 +-- 4 files changed, 71 insertions(+), 150 deletions(-) diff --git a/.gitignore b/.gitignore index 62bdea5a24..9be7cc094f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ docs/flytekit/flytekit.interfaces.html docs/searchindex.js docs/ __pycache__/ -/helm/charts/ \ No newline at end of file +/helm/charts/ +Chart.lock diff --git a/helm/README.MD b/helm/README.MD index 60a0acbcb1..90aa58c8eb 100644 --- a/helm/README.MD +++ b/helm/README.MD @@ -1,41 +1,24 @@ # flyte -![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.8.0](https://img.shields.io/badge/AppVersion-0.8.0-informational?style=flat-square) +![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.8.0](https://img.shields.io/badge/AppVersion-0.8.0-informational?style=flat-square) A Helm chart for Flyte -### SANDBOX INSTALLATION: -- [Install helm 3](https://helm.sh/docs/intro/install/) -- Fetch chart dependencies `helm dep up` -- Install Flyte sandbox: - -```bash -cd helm -helm install -n flyte -f values-sandbox.yaml --create-namespace flyte . -``` - -Customize your installation by changing settings in `values-sandbox.yaml`. -You can use the helm diff plugin to review any value changes you've made to your values: - -```bash -helm plugin install https://github.com/databus23/helm-diff -helm diff upgrade -f values-sandbox.yaml flyte . -``` +## Requirements -Then apply your changes: -```bash -helm upgrade -f values-sandbox.yaml flyte . -``` +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | contour | 4.1.2 | +| https://googlecloudplatform.github.io/spark-on-k8s-operator | sparkoperator(spark-operator) | 1.0.6 | +| https://kubernetes.github.io/dashboard/ | kubernetes-dashboard | 4.0.2 | -#### Alternative: Generate raw kubernetes yaml with helm template -- `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` +### SANDBOX INSTALLATION: +- Install `helm 3` realese version +- Generate `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` - Deploy the manifest `kubectl apply -f flyte_generated_sandbox.yaml` - - - When all pods are running - run end2end tests: `kubectl apply -f ../end2end/tests/endtoend.yaml` - Get flyte host `minikube service contour -n heptio-contour --url`. And then visit `http:///console` - ### CONFIGURATION NOTES: - The docker images, their tags and other default parameters are configured in `values.yaml` file. - Each Flyte installation type should have separate `values-*.yaml` file: for sandbox, EKS and etc. The configuration in `values.yaml` and the choosen config `values-*.yaml` are merged when generating the deployment manifest. @@ -45,157 +28,89 @@ helm upgrade -f values-sandbox.yaml flyte . | Key | Type | Default | Description | |-----|------|---------|-------------| +| cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"},{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | +| cluster_resource_manager.config.cluster_resources | object | `{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters | +| cluster_resource_manager.enabled | bool | `true` | Enables the Cluster resource manager component | +| cluster_resource_manager.templates | list | `[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"},{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}]` | Resource templates that should be applied | +| cluster_resource_manager.templates[0] | object | `{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"}` | Template for namespaces resources | +| cluster_resource_manager.templates[2] | object | `{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}` | Template for spark role | | common.databaseSecret.name | string | `""` | Specify name of K8s Secret which contains Database password. Leave it empty if you don't need this Secret | | common.databaseSecret.secretManifest | object | `{}` | Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets | -| common.flyteNamespaceTemplate.enabled | bool | `true` | | +| common.flyteNamespaceTemplate.enabled | bool | `false` | | +| common.ingress.albSSLRedirect | bool | `false` | | | common.ingress.annotations | object | `{}` | | | common.ingress.enabled | bool | `true` | | -| configmap.aa_namespace | string | `"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"` | Template for namespaces resources | -| configmap.ad_spark_role | string | `"apiVersion: rbac.authorization.k8s.io/v1beta1\nkind: Role\nmetadata:\n name: spark-role\n namespace: {{ namespace }}\nrules:\n- apiGroups:\n - \"\"\n resources:\n - pods\n verbs:\n - '*'\n- apiGroups:\n - \"\"\n resources:\n - services\n verbs:\n - '*'\n- apiGroups:\n - \"\"\n resources:\n - configmaps\n verbs:\n - '*'\n"` | Template for spark role | -| configmap.admin.admin.endpoint | string | `"flyteadmin:81"` | | -| configmap.admin.admin.insecure | bool | `true` | | -| configmap.admin.event.capacity | int | `1000` | | -| configmap.admin.event.rate | int | `500` | | -| configmap.admin.event.type | string | `"admin"` | | -| configmap.ae_spark_service_account | string | `"apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: spark\n namespace: {{ namespace }}\n"` | Template for spark SA | -| configmap.af_spark_role_binding | string | `"apiVersion: rbac.authorization.k8s.io/v1beta1\nkind: RoleBinding\nmetadata:\n name: spark-role-binding\n namespace: {{ namespace }} \nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: spark-role\nsubjects:\n- kind: ServiceAccount\n name: spark\n namespace: {{ namespace }}\n"` | Template for spark role binding | -| configmap.catalog.catalog-cache.endpoint | string | `"datacatalog:89"` | | -| configmap.catalog.catalog-cache.insecure | bool | `true` | | -| configmap.catalog.catalog-cache.type | string | `"datacatalog"` | | -| configmap.catalog_cache.plugins.catalogCache.reader.maxItems | int | `10000` | | -| configmap.catalog_cache.plugins.catalogCache.writer.maxItems | int | `10000` | | -| configmap.cluster_resources.cluster_resources.customData.development[0].projectQuotaCpu.value | string | `"4"` | | -| configmap.cluster_resources.cluster_resources.customData.development[1].projectQuotaMemory.value | string | `"3000Mi"` | | -| configmap.cluster_resources.cluster_resources.customData.production[0].projectQuotaCpu.value | string | `"5"` | | -| configmap.cluster_resources.cluster_resources.customData.production[1].projectQuotaMemory.value | string | `"4000Mi"` | | -| configmap.cluster_resources.cluster_resources.customData.staging[0].projectQuotaCpu.value | string | `"2"` | | -| configmap.cluster_resources.cluster_resources.customData.staging[1].projectQuotaMemory.value | string | `"3000Mi"` | | -| configmap.cluster_resources.cluster_resources.refresh | string | `"5m"` | | -| configmap.cluster_resources.cluster_resources.templatePath | string | `"/etc/flyte/clusterresource/templates"` | | +| common.ingress.separateGrpcIngress | bool | `false` | | +| common.ingress.separateGrpcIngressAnnotations."nginx.ingress.kubernetes.io/backend-protocol" | string | `"GRPC"` | | +| common.ingress.tls.enabled | bool | `true` | | +| common.ingress.webpackHMR | bool | `false` | | +| configmap.admin | object | `{"admin":{"endpoint":"flyteadmin:81","insecure":true},"event":{"capacity":1000,"rate":500,"type":"admin"}}` | Admin configuration | +| configmap.catalog | object | `{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}}` | Catalog configuration | | configmap.console.BASE_URL | string | `"/console"` | | | configmap.console.CONFIG_DIR | string | `"/etc/flyte/config"` | | -| configmap.copilot.plugins.k8s.co-pilot.image | string | `"docker.io/lyft/flytecopilot:v0.3.35"` | | -| configmap.copilot.plugins.k8s.co-pilot.name | string | `"flyte-copilot-"` | | -| configmap.copilot.plugins.k8s.co-pilot.start-timeout | string | `"30s"` | | -| configmap.core.propeller.downstream-eval-duration | string | `"30s"` | | -| configmap.core.propeller.enable-admin-launcher | bool | `true` | | -| configmap.core.propeller.leader-election.enabled | bool | `true` | | -| configmap.core.propeller.leader-election.lease-duration | string | `"15s"` | | -| configmap.core.propeller.leader-election.lock-config-map.name | string | `"propeller-leader"` | | -| configmap.core.propeller.leader-election.lock-config-map.namespace | string | `"flyte"` | | -| configmap.core.propeller.leader-election.renew-deadline | string | `"10s"` | | -| configmap.core.propeller.leader-election.retry-period | string | `"2s"` | | -| configmap.core.propeller.limit-namespace | string | `"all"` | | -| configmap.core.propeller.max-workflow-retries | int | `30` | | -| configmap.core.propeller.metadata-prefix | string | `"metadata/propeller"` | | -| configmap.core.propeller.metrics-prefix | string | `"flyte"` | | -| configmap.core.propeller.prof-port | int | `10254` | | -| configmap.core.propeller.queue.batch-size | int | `-1` | | -| configmap.core.propeller.queue.batching-interval | string | `"2s"` | | -| configmap.core.propeller.queue.queue.capacity | int | `100` | | -| configmap.core.propeller.queue.queue.rate | int | `10` | | -| configmap.core.propeller.queue.queue.type | string | `"bucket"` | | -| configmap.core.propeller.queue.sub-queue.capacity | int | `100` | | -| configmap.core.propeller.queue.sub-queue.rate | int | `10` | | -| configmap.core.propeller.queue.sub-queue.type | string | `"bucket"` | | -| configmap.core.propeller.queue.type | string | `"batch"` | | -| configmap.core.propeller.rawoutput-prefix | string | `"s3://my-s3-bucket/"` | | -| configmap.core.propeller.workers | int | `4` | | -| configmap.core.propeller.workflow-reeval-duration | string | `"30s"` | | +| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}}` | Core propeller configuration, follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller@v0.10.3/pkg/controller/config) | +| configmap.datacatalogServer.application.grpcPort | int | `8089` | | +| configmap.datacatalogServer.application.grpcServerReflection | bool | `true` | | +| configmap.datacatalogServer.application.httpPort | int | `8080` | | | configmap.datacatalogServer.datacatalog.metrics-scope | string | `"datacatalog"` | | | configmap.datacatalogServer.datacatalog.profiler-port | int | `10254` | | | configmap.datacatalogServer.datacatalog.storage-prefix | string | `"metadata/datacatalog"` | | -| configmap.db.database | object | `{}` | | | configmap.domain.domains[0].id | string | `"development"` | | | configmap.domain.domains[0].name | string | `"development"` | | | configmap.domain.domains[1].id | string | `"staging"` | | | configmap.domain.domains[1].name | string | `"staging"` | | | configmap.domain.domains[2].id | string | `"production"` | | | configmap.domain.domains[2].name | string | `"production"` | | -| configmap.enabled_plugins.tasks.max-plugin-phase-versions | int | `1000000` | | | configmap.enabled_plugins.tasks.task-plugins.enabled-plugins | list | `[]` | | | configmap.k8s.plugins.k8s.default-cpus | string | `"100m"` | | | configmap.k8s.plugins.k8s.default-env-vars | list | `[]` | | | configmap.k8s.plugins.k8s.default-memory | string | `"100Mi"` | | | configmap.logger.logger.level | int | `4` | | | configmap.logger.logger.show-source | bool | `true` | | -| configmap.qubole.plugins.qubole.quboleTokenKey | string | `"FLYTE_QUBOLE_CLIENT_TOKEN"` | | -| configmap.remote_data.remoteData.region | string | `"us-east-1"` | | -| configmap.remote_data.remoteData.scheme | string | `"local"` | | -| configmap.remote_data.remoteData.signedUrls.durationMinutes | int | `3` | | -| configmap.resource_manager.propeller.resourcemanager.redis.hostKey | string | `"mypassword"` | | -| configmap.resource_manager.propeller.resourcemanager.redis.hostPath | string | `"redis-resource-manager:6379"` | | -| configmap.resource_manager.propeller.resourcemanager.resourceMaxQuota | int | `10000` | | -| configmap.resource_manager.propeller.resourcemanager.type | string | `"redis"` | | -| configmap.server.flyteadmin.metadataStoragePrefix[0] | string | `"metadata"` | | -| configmap.server.flyteadmin.metadataStoragePrefix[1] | string | `"admin"` | | -| configmap.server.flyteadmin.metricsScope | string | `"flyte:"` | | -| configmap.server.flyteadmin.profilerPort | int | `10254` | | -| configmap.server.flyteadmin.roleNameKey | string | `"iam.amazonaws.com/role"` | | -| configmap.server.flyteadmin.testing.host | string | `"http://flyteadmin"` | | -| configmap.server.server.grpcPort | int | `8089` | | -| configmap.server.server.httpPort | int | `8088` | | -| configmap.server.server.security.allowCors | bool | `true` | | -| configmap.server.server.security.allowedHeaders[0] | string | `"Content-Type"` | | -| configmap.server.server.security.allowedOrigins[0] | string | `"*"` | | -| configmap.server.server.security.secure | bool | `false` | | -| configmap.server.server.security.useAuth | bool | `false` | | -| configmap.spark.plugins.spark.spark-config-default | list | `[{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]` | Spark default configuration | -| configmap.storage.storage.connection.auth-type | string | `"accesskey"` | | -| configmap.storage.storage.connection.region | string | `"us-east-1"` | | -| configmap.storage.storage.container | string | `"my-s3-bucket"` | | -| configmap.storage.storage.type | string | `"minio"` | | +| configmap.resource_manager.propeller | object | `{"resourcemanager":{"redis":{"hostKey":"mypassword","hostPath":"redis-resource-manager:6379"},"resourceMaxQuota":10000,"type":"redis"}}` | resource manager configuration | +| configmap.server | object | `{"flyteadmin":{"eventVersion":1,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"secure":false,"useAuth":false}}}` | Flyteadmin server config | | configmap.task_logs.plugins.logs.kubernetes-enabled | bool | `false` | | -| configmap.task_resource_defaults.task_resources.defaults.cpu | string | `"100m"` | | -| configmap.task_resource_defaults.task_resources.defaults.memory | string | `"1000Mi"` | | -| configmap.task_resource_defaults.task_resources.defaults.storage | string | `"5Mi"` | | -| configmap.task_resource_defaults.task_resources.limits.cpu | string | `"15000m"` | | -| configmap.task_resource_defaults.task_resources.limits.memory | string | `"100Gi"` | | -| configmap.task_resource_defaults.task_resources.limits.storage | string | `"400Gi"` | | +| configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}}` | Task default resources configuration | +| configmap.task_resource_defaults.task_resources | object | `{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}` | Task default resources parameters | | contour.affinity | object | `{}` | affinity for Contour deployment | -| contour.contour.image.pullPolicy | string | `"IfNotPresent"` | | -| contour.contour.image.repository | string | `"gcr.io/heptio-images/contour"` | Docker image for Contour | -| contour.contour.image.tag | string | `"v0.6.1"` | | | contour.contour.resources | object | `{"limits":{"cpu":"100m","memory":"100Mi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Contour | | contour.enabled | bool | `true` | | -| contour.envoy.image.pullPolicy | string | `"IfNotPresent"` | | -| contour.envoy.image.repository | string | `"docker.io/envoyproxy/envoy-alpine"` | Docker image for Envoy | -| contour.envoy.image.tag | string | `"v1.6.0"` | | | contour.envoy.resources | object | `{"limits":{"cpu":"100m","memory":"100Mi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Envoy | | contour.nodeSelector | object | `{}` | nodeSelector for Contour deployment | | contour.podAnnotations | object | `{}` | Annotations for Contour pods | | contour.replicaCount | int | `1` | Replicas count for Contour deployment | -| contour.service | object | `{"annotations":{},"ports":[{"nodePort":30081,"port":80,"protocol":"TCP"}],"type":"NodePort"}` | Service settings for Contour | | contour.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Contour pods | | contour.tolerations | list | `[]` | tolerations for Contour deployment | | datacatalog.affinity | object | `{}` | affinity for Datacatalog deployment | | datacatalog.configPath | string | `"/etc/datacatalog/config/*.yaml"` | Default regex string for searching configuration files | | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | | -| datacatalog.image.repository | string | `"docker.io/lyft/datacatalog"` | Docker image for Datacatalog deployment | -| datacatalog.image.tag | string | `"v0.2.2"` | | +| datacatalog.image.repository | string | `"ghcr.io/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | +| datacatalog.image.tag | string | `"v0.3.0"` | | | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | | datacatalog.resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Datacatalog deployment | -| datacatalog.service | object | `{"annotations":{"contour.heptio.com/upstream-protocol.h2c":"grpc"},"type":"NodePort"}` | Service settings for Datacatalog | +| datacatalog.service | object | `{"annotations":{"alb.ingress.kubernetes.io/backend-protocol-version":"GRPC","projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"}` | Service settings for Datacatalog | | datacatalog.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Datacatalog pods | | datacatalog.tolerations | list | `[]` | tolerations for Datacatalog deployment | +| db.database | object | `{}` | | | flyteadmin.affinity | object | `{}` | affinity for Flyteadmin deployment | | flyteadmin.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | | -| flyteadmin.image.repository | string | `"docker.io/lyft/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyteadmin.image.tag | string | `"v0.3.7"` | | +| flyteadmin.image.repository | string | `"ghcr.io/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | +| flyteadmin.image.tag | string | `"v0.4.0"` | | | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | | flyteadmin.replicaCount | int | `1` | Replicas count for Flyteadmin deployment | | flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment | -| flyteadmin.service | object | `{"annotations":{"contour.heptio.com/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"}` | Service settings for Flyteadmin | +| flyteadmin.service | object | `{"annotations":{"alb.ingress.kubernetes.io/backend-protocol-version":"GRPC","projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin | | flyteadmin.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Flyteadmin pods | | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment | | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | | -| flyteconsole.image.repository | string | `"docker.io/lyft/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyteconsole.image.tag | string | `"v0.12.1"` | | +| flyteconsole.image.repository | string | `"ghcr.io/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | +| flyteconsole.image.tag | string | `"v0.19.6"` | | | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | | flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment | @@ -203,21 +118,23 @@ helm upgrade -f values-sandbox.yaml flyte . | flyteconsole.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Flyteconsole | | flyteconsole.tolerations | list | `[]` | tolerations for Flyteconsole deployment | | flytepropeller.affinity | object | `{}` | affinity for Flytepropeller deployment | +| flytepropeller.cacheSizeMbs | int | `0` | | | flytepropeller.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | | -| flytepropeller.image.repository | string | `"docker.io/lyft/flytepropeller"` | Docker image for Flytepropeller deployment | -| flytepropeller.image.tag | string | `"v0.4.2"` | | +| flytepropeller.image.repository | string | `"ghcr.io/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | +| flytepropeller.image.tag | string | `"v0.7.8"` | | | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | | flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytepropeller deployment | | flytepropeller.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Flytepropeller pods | | flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment | +| kubernetes-dashboard.enabled | bool | `false` | | | minio.affinity | object | `{}` | affinity for Minio deployment | | minio.enabled | bool | `true` | | | minio.image.pullPolicy | string | `"IfNotPresent"` | | | minio.image.repository | string | `"minio/minio"` | Docker image for Minio deployment | -| minio.image.tag | string | `"RELEASE.2019-06-04T01-15-58Z"` | | +| minio.image.tag | string | `"RELEASE.2020-12-16T05-05-17Z"` | | | minio.nodeSelector | object | `{}` | nodeSelector for Minio deployment | | minio.podAnnotations | object | `{}` | Annotations for Minio pods | | minio.replicaCount | int | `1` | Replicas count for Minio deployment | @@ -228,7 +145,7 @@ helm upgrade -f values-sandbox.yaml flyte . | postgres.enabled | bool | `true` | | | postgres.image.pullPolicy | string | `"IfNotPresent"` | | | postgres.image.repository | string | `"postgres"` | Docker image for Postgres deployment | -| postgres.image.tag | string | `"10.1"` | | +| postgres.image.tag | string | `"10.16"` | | | postgres.nodeSelector | object | `{}` | nodeSelector for Postgres deployment | | postgres.podAnnotations | object | `{}` | Annotations for Postgres pods | | postgres.replicaCount | int | `1` | Replicas count for Postgres deployment | @@ -258,15 +175,18 @@ helm upgrade -f values-sandbox.yaml flyte . | redis.resources | object | `{"limits":{"cpu":"1000m","memory":"1Gi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Redis Statefulset | | redis.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Redis | | redis.tolerations | list | `[]` | tolerations for Redis Statefulset | -| sparkoperator.affinity | object | `{}` | affinity for Sparkoperator deployment | | sparkoperator.enabled | bool | `true` | | -| sparkoperator.image.pullPolicy | string | `"IfNotPresent"` | | -| sparkoperator.image.repository | string | `"gcr.io/spark-operator/spark-operator"` | Docker image for Sparkoperator | -| sparkoperator.image.tag | string | `"v2.4.0-v1beta1-0.9.0"` | | -| sparkoperator.nodeSelector | object | `{}` | nodeSelector for Sparkoperator deployment | -| sparkoperator.podAnnotations | object | `{}` | Annotations for Sparkoperator pods | +| sparkoperator.image.tag | string | `"v1beta2-1.2.0-3.0.0"` | Docker image for Sparkoperator | +| sparkoperator.plugin_config | object | `{"plugins":{"spark":{"spark-config-default":[{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]}}}` | Spark plugin configuration | +| sparkoperator.plugin_config.plugins.spark.spark-config-default | list | `[{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]` | Spark default configuration | | sparkoperator.replicaCount | int | `1` | Replicas count for Sparkoperator deployment | | sparkoperator.resources | object | `{"limits":{"cpu":"1000m","memory":"500M"},"requests":{"cpu":"10m","memory":"50M"}}` | Default resources requests and limits for Sparkoperator | -| sparkoperator.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Contour | -| sparkoperator.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Sparkoperator pods | -| sparkoperator.tolerations | list | `[]` | tolerations for Sparkoperator deployment | +| storage.bucketName | string | `"my-s3-bucket"` | bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. | +| storage.custom | object | `{}` | GCP project ID. Required for storage type gcs. projectId: -- Settings for storage type custom. See https://github:com/graymeta/stow for supported storage providers/settings. | +| storage.gcs | string | `nil` | settings for storage type gcs | +| storage.s3 | object | `{"region":"us-east-1"}` | settings for storage type s3 | +| storage.type | string | `"sandbox"` | Sets the storage type. Supported values are sandbox, s3, gcs and custom. | +| workflow_notifications.config | object | `{}` | | +| workflow_notifications.enabled | bool | `false` | | +| workflow_scheduler.config | object | `{}` | | +| workflow_scheduler.enabled | bool | `false` | | diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 6db61dd2ba..5d8527739b 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -5,7 +5,7 @@ flyteadmin: replicaCount: 2 - # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + # -- IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html serviceAccountAnnotations: eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte @@ -150,12 +150,12 @@ common: ingress: host: flyte.example.com annotations: - # aws-load-balancer-controller v2.1 or higher is required - https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.1/ + # -- aws-load-balancer-controller v2.1 or higher is required - https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.1/ kubernetes.io/ingress.class: alb alb.ingress.kubernetes.io/tags: service_instance=production alb.ingress.kubernetes.io/scheme: internet-facing alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - # Instruct ALB Controller to not create multiple load balancers (and hence maintain a single endpoint for both GRPC and Http) + # -- Instruct ALB Controller to not create multiple load balancers (and hence maintain a single endpoint for both GRPC and Http) alb.ingress.kubernetes.io/group.name: flytesystem # Replace certificate Arn with one deployed to your EKS cluster. Follow instructions in README.md alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 diff --git a/helm/values.yaml b/helm/values.yaml index 8bc0675b75..f5f532d5ea 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -438,7 +438,7 @@ configmap: httpPort: 8080 grpcServerReflection: true - # Task default resources configuration + # -- Task default resources configuration task_resource_defaults: # -- Task default resources parameters task_resources: @@ -452,7 +452,7 @@ configmap: storage: 20Mi gpu: 1 - # Admin configuration + # -- Admin configuration admin: event: type: admin @@ -462,14 +462,14 @@ configmap: endpoint: flyteadmin:81 insecure: true - # Catalog configuration + # -- Catalog configuration catalog: catalog-cache: endpoint: datacatalog:89 type: datacatalog insecure: true - # Copilot configuration + # -- Copilot configuration copilot: plugins: k8s: @@ -478,7 +478,7 @@ configmap: image: ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2 start-timeout: 30s - # Core propeller configuration + # -- Core propeller configuration, follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller@v0.10.3/pkg/controller/config) core: propeller: rawoutput-prefix: s3://my-s3-bucket/ @@ -678,7 +678,7 @@ sparkoperator: cpu: 10m memory: 50M - # Spark plugin configuration + # -- Spark plugin configuration plugin_config: plugins: spark: From c7d0470ea66ed32e1858315172e7a04ce866ed66 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 20 Apr 2021 11:32:55 -0700 Subject: [PATCH 27/92] more docs Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- helm/README.MD | 46 ++++++++++++++++++-------------------------- helm/values.yaml | 50 +++++++++++++++++++++++++++--------------------- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/helm/README.MD b/helm/README.MD index 90aa58c8eb..fa2a3694b8 100644 --- a/helm/README.MD +++ b/helm/README.MD @@ -29,6 +29,7 @@ A Helm chart for Flyte | Key | Type | Default | Description | |-----|------|---------|-------------| | cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"},{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | +| cluster_resource_manager.config | object | `{"cluster_resources":{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}}` | Configmap for ClusterResource parameters | | cluster_resource_manager.config.cluster_resources | object | `{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters | | cluster_resource_manager.enabled | bool | `true` | Enables the Cluster resource manager component | | cluster_resource_manager.templates | list | `[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"},{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}]` | Resource templates that should be applied | @@ -44,33 +45,25 @@ A Helm chart for Flyte | common.ingress.separateGrpcIngressAnnotations."nginx.ingress.kubernetes.io/backend-protocol" | string | `"GRPC"` | | | common.ingress.tls.enabled | bool | `true` | | | common.ingress.webpackHMR | bool | `false` | | -| configmap.admin | object | `{"admin":{"endpoint":"flyteadmin:81","insecure":true},"event":{"capacity":1000,"rate":500,"type":"admin"}}` | Admin configuration | -| configmap.catalog | object | `{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}}` | Catalog configuration | -| configmap.console.BASE_URL | string | `"/console"` | | -| configmap.console.CONFIG_DIR | string | `"/etc/flyte/config"` | | +| configmap.admin | object | `{"admin":{"endpoint":"flyteadmin:81","insecure":true},"event":{"capacity":1000,"rate":500,"type":"admin"}}` | Admin Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/subworkflow/launchplan#AdminConfig) | +| configmap.catalog | object | `{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}}` | Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) | +| configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | | configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}}` | Core propeller configuration, follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller@v0.10.3/pkg/controller/config) | -| configmap.datacatalogServer.application.grpcPort | int | `8089` | | -| configmap.datacatalogServer.application.grpcServerReflection | bool | `true` | | -| configmap.datacatalogServer.application.httpPort | int | `8080` | | -| configmap.datacatalogServer.datacatalog.metrics-scope | string | `"datacatalog"` | | -| configmap.datacatalogServer.datacatalog.profiler-port | int | `10254` | | -| configmap.datacatalogServer.datacatalog.storage-prefix | string | `"metadata/datacatalog"` | | -| configmap.domain.domains[0].id | string | `"development"` | | -| configmap.domain.domains[0].name | string | `"development"` | | -| configmap.domain.domains[1].id | string | `"staging"` | | -| configmap.domain.domains[1].name | string | `"staging"` | | -| configmap.domain.domains[2].id | string | `"production"` | | -| configmap.domain.domains[2].name | string | `"production"` | | -| configmap.enabled_plugins.tasks.task-plugins.enabled-plugins | list | `[]` | | -| configmap.k8s.plugins.k8s.default-cpus | string | `"100m"` | | -| configmap.k8s.plugins.k8s.default-env-vars | list | `[]` | | -| configmap.k8s.plugins.k8s.default-memory | string | `"100Mi"` | | -| configmap.logger.logger.level | int | `4` | | -| configmap.logger.logger.show-source | bool | `true` | | +| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}}` | Core propeller configuration | +| configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | +| configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | +| configmap.domain | object | `{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]}` | Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. | +| configmap.enabled_plugins.tasks | object | `{"task-plugins":{"enabled-plugins":[]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) | +| configmap.enabled_plugins.tasks.task-plugins | object | `{"enabled-plugins":[]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) | +| configmap.enabled_plugins.tasks.task-plugins.enabled-plugins | list | `[]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config) | +| configmap.k8s | object | `{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}}}` | Kubernetes specific Flyte configuration | +| configmap.k8s.plugins.k8s | object | `{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}` | Configuration section for all K8s specific plugins [Configuration structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/flytek8s/config) | +| configmap.logger | object | `{"logger":{"level":4,"show-source":true}}` | Logger configuration | +| configmap.resource_manager | object | `{"propeller":{"resourcemanager":{"redis":{"hostKey":"mypassword","hostPath":"redis-resource-manager:6379"},"resourceMaxQuota":10000,"type":"redis"}}}` | Resource manager configuration | | configmap.resource_manager.propeller | object | `{"resourcemanager":{"redis":{"hostKey":"mypassword","hostPath":"redis-resource-manager:6379"},"resourceMaxQuota":10000,"type":"redis"}}` | resource manager configuration | -| configmap.server | object | `{"flyteadmin":{"eventVersion":1,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"secure":false,"useAuth":false}}}` | Flyteadmin server config | -| configmap.task_logs.plugins.logs.kubernetes-enabled | bool | `false` | | +| configmap.server | object | `{"flyteadmin":{"eventVersion":1,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | +| configmap.task_logs | object | `{"plugins":{"logs":{"kubernetes-enabled":false}}}` | Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various logging engines | | configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}}` | Task default resources configuration | | configmap.task_resource_defaults.task_resources | object | `{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}` | Task default resources parameters | | contour.affinity | object | `{}` | affinity for Contour deployment | @@ -186,7 +179,6 @@ A Helm chart for Flyte | storage.gcs | string | `nil` | settings for storage type gcs | | storage.s3 | object | `{"region":"us-east-1"}` | settings for storage type s3 | | storage.type | string | `"sandbox"` | Sets the storage type. Supported values are sandbox, s3, gcs and custom. | -| workflow_notifications.config | object | `{}` | | -| workflow_notifications.enabled | bool | `false` | | +| workflow_notifications | object | `{"config":{},"enabled":false}` | **Optional Component** Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of various events in their workflows | | workflow_scheduler.config | object | `{}` | | | workflow_scheduler.enabled | bool | `false` | | diff --git a/helm/values.yaml b/helm/values.yaml index f5f532d5ea..6b3871a522 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -392,12 +392,12 @@ db: # configmap: - # Configmap for Flyte console UI + # -- Configuration for Flyte console UI console: BASE_URL: /console CONFIG_DIR: /etc/flyte/config - # Domains configuration for Flyte projects + # -- Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. domain: domains: - id: development @@ -407,7 +407,7 @@ configmap: - id: production name: production - # -- Flyteadmin server config + # -- FlyteAdmin server configuration server: server: httpPort: 8088 @@ -427,7 +427,7 @@ configmap: testing: host: http://flyteadmin - # Datacatalog server config + # -- Datacatalog server config datacatalogServer: datacatalog: storage-prefix: metadata/datacatalog @@ -452,7 +452,7 @@ configmap: storage: 20Mi gpu: 1 - # -- Admin configuration + # -- Admin Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/subworkflow/launchplan#AdminConfig) admin: event: type: admin @@ -462,7 +462,8 @@ configmap: endpoint: flyteadmin:81 insecure: true - # -- Catalog configuration + # -- Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) + # Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) catalog: catalog-cache: endpoint: datacatalog:89 @@ -473,13 +474,15 @@ configmap: copilot: plugins: k8s: + # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) co-pilot: name: flyte-copilot- image: ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2 start-timeout: 30s - # -- Core propeller configuration, follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller@v0.10.3/pkg/controller/config) + # -- Core propeller configuration core: + # -- follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). propeller: rawoutput-prefix: s3://my-s3-bucket/ metadata-prefix: metadata/propeller @@ -513,30 +516,34 @@ configmap: type: bucket rate: 10 capacity: 100 + # -- For Workflow store use configuration [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/workflowstore#Config) + # - # Plugins configuration enabled_plugins: + # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) tasks: + # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) task-plugins: + # -- [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config) enabled-plugins: [] - # Kubernetes plugin configuration + # -- Kubernetes specific Flyte configuration k8s: plugins: + # -- Configuration section for all K8s specific plugins [Configuration structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/flytek8s/config) k8s: default-env-vars: [] - # - DEFAULT_ENV_VAR: VALUE - + # DEFAULT_ENV_VAR: VALUE default-cpus: 100m default-memory: 100Mi - # Logger configuration + # -- Logger configuration logger: logger: show-source: true level: 4 - # Resource manager configuration + # -- Resource manager configuration resource_manager: # -- resource manager configuration propeller: @@ -547,7 +554,9 @@ configmap: hostPath: redis-resource-manager:6379 hostKey: mypassword - # Tasks logs plugin configuration + # -- Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. + # Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various + # logging engines task_logs: plugins: logs: @@ -559,16 +568,16 @@ configmap: # All these features are optional, but are critical to run certain features # ------------------------------------------------------------------------ -# -- +# -- **Optional Component** # Flyte uses a cloud hosted Cron scheduler to run workflows on a schedule. The following module is optional. Without, -# this module, you will not have scheduled launchplans/workflows. +# this module, you will not have scheduled launchplans / workflows. workflow_scheduler: enabled: false config: {} -# -- +# -- **Optional Component** # Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of # various events in their workflows workflow_notifications: @@ -582,7 +591,7 @@ workflow_notifications: cluster_resource_manager: # -- Enables the Cluster resource manager component enabled: true - # Configmap for ClusterResource parameters + # -- Configmap for ClusterResource parameters config: # -- ClusterResource parameters cluster_resources: @@ -657,10 +666,7 @@ cluster_resource_manager: # Optional Plugins # -------------------------------------------------------- -# -# SPARK Plugin using the Spark Operator -# - +# -- Optional: Spark Plugin using the Spark Operator sparkoperator: # --- enable or disable Sparkoperator deployment installation enabled: true From 040a7f6428b07d27810d5bcd3ba5677bc7a05aa1 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Thu, 22 Apr 2021 15:56:44 -0700 Subject: [PATCH 28/92] Add references to flyteadmin config Signed-off-by: Katrina Rogan Signed-off-by: Haytham Abuelfutuh --- helm/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm/values.yaml b/helm/values.yaml index 6b3871a522..9acb1ff6d5 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -409,6 +409,7 @@ configmap: # -- FlyteAdmin server configuration server: + # Refer to the [server config](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/config#ServerConfig). server: httpPort: 8088 grpcPort: 8089 @@ -416,6 +417,7 @@ configmap: secure: false useAuth: false allowCors: true + # Refer to the full [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ApplicationConfig) for documentation. flyteadmin: roleNameKey: "iam.amazonaws.com/role" profilerPort: 10254 @@ -439,6 +441,7 @@ configmap: grpcServerReflection: true # -- Task default resources configuration + # Refer to the full [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#TaskResourceConfiguration). task_resource_defaults: # -- Task default resources parameters task_resources: @@ -594,6 +597,7 @@ cluster_resource_manager: # -- Configmap for ClusterResource parameters config: # -- ClusterResource parameters + # Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" customData: From 3a14808692d3886e146f268fedf46fcd1d67e213 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Wed, 21 Apr 2021 15:30:49 -0700 Subject: [PATCH 29/92] Improved value.yaml docs Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- helm/values-eks.yaml | 50 +++++++++++++++++++++++++++++++++++++++++++- helm/values.yaml | 41 ++++++++++++++++++++++++++++-------- 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 5d8527739b..2d0dd9bb7d 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -259,25 +259,50 @@ configmap: capacity: 1000 enabled_plugins: + # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) tasks: + # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) task-plugins: + # -- [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend + # plugins enabled-plugins: - container - sidecar - spark - k8s-array - pytorch + - sagemaker_hyperparameter_tuning + - sagemaker_custom_training + - sagemaker_training + - athena default-for-task-types: container: container sidecar: sidecar spark: spark container_array: k8s-array pytorch: pytorch + sagemaker_custom_training_task: sagemaker_custom_training + sagemaker_custom_training_job_task: sagemaker_custom_training + hive: athena logger: logger: level: 5 + # -- Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. + # Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various + # logging engines + task_logs: + plugins: + logs: + kubernetes-enabled: false + # -- One option is to enable cloudwatch logging for EKS, update the region and log group accordingly + cloudwatch-enabled: true + # -- region where logs are hosted + cloudwatch-region: us-east-2 + # -- cloudwatch log-group + cloudwatch-log-group: /aws/containerinsights/flyte-demo-2/application + # ---------------------------------------------------------------- # Optional Modules @@ -461,4 +486,27 @@ pytorchoperator: memory: 1000Mi requests: cpu: 50m - memory: 250Mi \ No newline at end of file + memory: 250Mi + +# ---------------------- +# -- Distributed tensorflow training using the TF-Operator. This can be installed as explained [here](https://github.com/kubeflow/tf-operator) +# There is no helm chart maintained and hence it might need an independent installation +tf_operator: + enabled: true + + +# ----------------- +# -- Training on AWS Sagemaker using AWS Sagemaker operator. To actually install the operator, please follow instructions [here](https://github.com/aws/amazon-sagemaker-operator-for-k8s/tree/master/hack/charts/installer/rolebased) +# Use the config section here to just enable sagemaker plugin in Flyte, after you have installed the operator using the information +sagemaker: + enabled: true + plugin_config: + plugins: + sagemaker: + roleArn: arn:aws:iam::590375264460:role/eksctl-flyte-demo-2-addon-iamserviceaccount-Role1-62EXWRIT27CA + region: us-east-2 + + +# --------------------------- +# Flink jobs using the Flink Operator +# diff --git a/helm/values.yaml b/helm/values.yaml index 9acb1ff6d5..1d6e64a660 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -527,8 +527,16 @@ configmap: tasks: # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) task-plugins: - # -- [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config) - enabled-plugins: [] + # -- [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend + # plugins + enabled-plugins: + - container + - sidecar + - k8s-array + default-for-task-types: + container: container + sidecar: sidecar + container_array: k8s-array # -- Kubernetes specific Flyte configuration k8s: @@ -564,6 +572,12 @@ configmap: plugins: logs: kubernetes-enabled: false + # -- One option is to enable cloudwatch logging for EKS, update the region and log group accordingly + cloudwatch-enabled: true + # -- region where logs are hosted + cloudwatch-region: us-east-2 + # -- cloudwatch log-group + cloudwatch-log-group: /aws/containerinsights/flyte-demo-2/application # ---------------------------------------------------------------- # Optional Modules @@ -747,14 +761,23 @@ pytorchoperator: affinity: {} # ---------------------- -# Distributed tensorflow training using the TF-Operator -# +# -- Distributed tensorflow training using the TF-Operator. This can be installed as explained [here](https://github.com/kubeflow/tf-operator) +# There is no helm chart maintained and hence it might need an independent installation +tf_operator: + enabled: false -# ---------------------- -# Training on AWS Sagemaker using AWS Sagemaker operator -# +# ----------------- +# -- Training on AWS Sagemaker using AWS Sagemaker operator. To actually install the operator, please follow instructions [here](https://github.com/aws/amazon-sagemaker-operator-for-k8s/tree/master/hack/charts/installer/rolebased) +# Use the config section here to just enable sagemaker plugin in Flyte, after you have installed the operator using the information +sagemaker: + enabled: false + plugin_config: + plugins: + sagemaker: + roleArn: + region: + # --------------------------- -# Flink jobs using the Flink Operator -# +# -- Flink jobs using the Flink Operator: this is work in progress From 6235ead3e5405508ebd29fd56916fe61863a3da2 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Wed, 21 Apr 2021 16:03:15 -0700 Subject: [PATCH 30/92] updated yaml Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- helm/templates/common/ingress.yaml | 157 ++++++++++++++++++++--------- helm/values-eks.yaml | 32 +++--- helm/values.yaml | 2 +- 3 files changed, 125 insertions(+), 66 deletions(-) diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index f5087bf9f3..d1aaa3b114 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -1,5 +1,5 @@ {{- if .Values.common.ingress.enabled }} -apiVersion: networking.k8s.io/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ template "flyte.name" . }} @@ -15,122 +15,166 @@ spec: - path: /* pathType: ImplementationSpecific backend: - serviceName: ssl-redirect - servicePort: use-annotation + service: + name: ssl-redirect + port: + name: use-annotation {{- end }} # This is useful only for frontend development {{- if .Values.common.ingress.webpackHMR }} - path: /__webpack_hmr pathType: ImplementationSpecific backend: - serviceName: flyteconsole - servicePort: 80 + service: + name: flyteconsole + port: + number: 80 {{- end }} # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 87 + service: + name: flyteadmin + port: + number: 87 # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml - path: /console pathType: ImplementationSpecific backend: - serviceName: flyteconsole - servicePort: 80 + service: + name: flyteconsole + port: + number: 80 - path: /console/* pathType: ImplementationSpecific backend: - serviceName: flyteconsole - servicePort: 80 + service: + name: flyteconsole + port: + number: 80 - path: /api pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /api/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /healthcheck pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /v1/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /.well-known/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /login pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /login/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /logout pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /logout/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /callback pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /callback/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /me pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /config pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 - path: /config/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 {{- if not .Values.common.ingress.separateGrpcIngress }} # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. - path: /flyteidl.service.AdminService pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 81 + service: + name: flyteadmin + port: + number: 81 - path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 81 + service: + name: flyteadmin + port: + number: 81 {{- end }} {{- with .Values.common.ingress.host }} host: {{ . }} @@ -145,7 +189,7 @@ spec: # Certain ingress controllers like nginx cannot serve HTTP 1 and GRPC with a single ingress because GRPC can only # enabled on the ingress object, not on backend services (GRPC annotation is set on the ingress, not on the services). --- -apiVersion: networking.k8s.io/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ template "flyte.name" . }}-grpc @@ -161,17 +205,30 @@ spec: rules: - http: paths: + {{- if .Values.common.ingress.albSSLRedirect }} + - backend: + service: + name: ssl-redirect + port: + name: ssl-redirect + path: /* + pathType: ImplementationSpecific + {{- end }} # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. - path: /flyteidl.service.AdminService pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 81 + service: + name: flyteadmin + port: + number: 81 - path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific backend: - serviceName: flyteadmin - servicePort: 81 + service: + name: flyteadmin + port: + number: 81 {{- with .Values.common.ingress.host }} host: {{ . }} {{- end }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 2d0dd9bb7d..fb22db05fe 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -77,13 +77,14 @@ flytepropeller: eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte resources: limits: - cpu: 500m - ephemeral-storage: 200Mi - memory: 500Mi + cpu: 1 + ephemeral-storage: 1GB + memory: 2GB requests: - cpu: 50m - ephemeral-storage: 200Mi - memory: 200Mi + cpu: 1 + ephemeral-storage: 1GB + memory: 2GB + cacheSizeMbs: 512 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -148,18 +149,19 @@ contour: common: ingress: - host: flyte.example.com + host: demo2.nuclyde.io + albSSLRedirect: true annotations: # -- aws-load-balancer-controller v2.1 or higher is required - https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.1/ + # For EKS if using [ALB](https://kubernetes-sigs.github.io/aws-load-balancer-controller/guide/ingress/annotations/), these annotations are set kubernetes.io/ingress.class: alb alb.ingress.kubernetes.io/tags: service_instance=production alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:590375264460:certificate/e2f04275-2dff-4118-a493-ed3ec8f41605 + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' # -- Instruct ALB Controller to not create multiple load balancers (and hence maintain a single endpoint for both GRPC and Http) alb.ingress.kubernetes.io/group.name: flytesystem - # Replace certificate Arn with one deployed to your EKS cluster. Follow instructions in README.md - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' databaseSecret: name: db-pass secretManifest: @@ -197,16 +199,16 @@ storage: # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. type: s3 # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. - bucketName: + bucketName: flyte-demo s3: - region: us-east-1 + region: us-east-2 db: database: port: 5432 - username: postgres - host: - dbname: flyte-development + username: flyte + host: flyteadmin.cluster-cguibupmpitq.us-east-2.rds.amazonaws.com + dbname: flyteadmin passwordPath: /etc/db/pass.txt # diff --git a/helm/values.yaml b/helm/values.yaml index 1d6e64a660..2c45dd1225 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -780,4 +780,4 @@ sagemaker: # --------------------------- -# -- Flink jobs using the Flink Operator: this is work in progress +# -- Flink jobs using the Flink Operator: this is work in progress From 1cd0ce83c0e09f5047d063bd7e121a9dfa177344 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Fri, 23 Apr 2021 13:51:41 -0700 Subject: [PATCH 31/92] Updated working basic eks config Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- helm/README.MD | 29 ++++-- helm/templates/admin/rbac.yaml | 4 +- helm/templates/admin/service.yaml | 27 +++--- helm/templates/datacatalog/rbac.yaml | 2 + helm/templates/propeller/rbac.yaml | 3 +- helm/values-eks.yaml | 136 +++++++++++++++------------ helm/values.yaml | 28 ++++-- 7 files changed, 142 insertions(+), 87 deletions(-) diff --git a/helm/README.MD b/helm/README.MD index fa2a3694b8..2c1e8c9fb4 100644 --- a/helm/README.MD +++ b/helm/README.MD @@ -47,23 +47,26 @@ A Helm chart for Flyte | common.ingress.webpackHMR | bool | `false` | | | configmap.admin | object | `{"admin":{"endpoint":"flyteadmin:81","insecure":true},"event":{"capacity":1000,"rate":500,"type":"admin"}}` | Admin Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/subworkflow/launchplan#AdminConfig) | | configmap.catalog | object | `{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}}` | Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) | -| configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | +| configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config","DISABLE_AUTH":"1"}` | Configuration for Flyte console UI | | configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | | configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | | configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}}` | Core propeller configuration | | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | | configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | | configmap.domain | object | `{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]}` | Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. | -| configmap.enabled_plugins.tasks | object | `{"task-plugins":{"enabled-plugins":[]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) | -| configmap.enabled_plugins.tasks.task-plugins | object | `{"enabled-plugins":[]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) | -| configmap.enabled_plugins.tasks.task-plugins.enabled-plugins | list | `[]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config) | +| configmap.enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) | +| configmap.enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) | +| configmap.enabled_plugins.tasks.task-plugins.enabled-plugins | list | `["container","sidecar","k8s-array"]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend plugins | | configmap.k8s | object | `{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}}}` | Kubernetes specific Flyte configuration | | configmap.k8s.plugins.k8s | object | `{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}` | Configuration section for all K8s specific plugins [Configuration structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/flytek8s/config) | | configmap.logger | object | `{"logger":{"level":4,"show-source":true}}` | Logger configuration | | configmap.resource_manager | object | `{"propeller":{"resourcemanager":{"redis":{"hostKey":"mypassword","hostPath":"redis-resource-manager:6379"},"resourceMaxQuota":10000,"type":"redis"}}}` | Resource manager configuration | | configmap.resource_manager.propeller | object | `{"resourcemanager":{"redis":{"hostKey":"mypassword","hostPath":"redis-resource-manager:6379"},"resourceMaxQuota":10000,"type":"redis"}}` | resource manager configuration | | configmap.server | object | `{"flyteadmin":{"eventVersion":1,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | -| configmap.task_logs | object | `{"plugins":{"logs":{"kubernetes-enabled":false}}}` | Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various logging engines | +| configmap.task_logs | object | `{"plugins":{"logs":{"cloudwatch-enabled":true,"cloudwatch-log-group":"/aws/containerinsights/flyte-demo-2/application","cloudwatch-region":"us-east-2","kubernetes-enabled":false}}}` | Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various logging engines | +| configmap.task_logs.plugins.logs.cloudwatch-enabled | bool | `true` | One option is to enable cloudwatch logging for EKS, update the region and log group accordingly | +| configmap.task_logs.plugins.logs.cloudwatch-log-group | string | `"/aws/containerinsights/flyte-demo-2/application"` | cloudwatch log-group | +| configmap.task_logs.plugins.logs.cloudwatch-region | string | `"us-east-2"` | region where logs are hosted | | configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}}` | Task default resources configuration | | configmap.task_resource_defaults.task_resources | object | `{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}` | Task default resources parameters | | contour.affinity | object | `{}` | affinity for Contour deployment | @@ -85,7 +88,9 @@ A Helm chart for Flyte | datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | | datacatalog.resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Datacatalog deployment | | datacatalog.service | object | `{"annotations":{"alb.ingress.kubernetes.io/backend-protocol-version":"GRPC","projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"}` | Service settings for Datacatalog | -| datacatalog.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Datacatalog pods | +| datacatalog.serviceAccount | object | `{"annotations":{},"create":true}` | Configuration for service accounts for Datacatalog | +| datacatalog.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Datacatalog pods | +| datacatalog.serviceAccount.create | bool | `true` | Should a service account be created for Datacatalog | | datacatalog.tolerations | list | `[]` | tolerations for Datacatalog deployment | | db.database | object | `{}` | | | flyteadmin.affinity | object | `{}` | affinity for Flyteadmin deployment | @@ -98,7 +103,9 @@ A Helm chart for Flyte | flyteadmin.replicaCount | int | `1` | Replicas count for Flyteadmin deployment | | flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment | | flyteadmin.service | object | `{"annotations":{"alb.ingress.kubernetes.io/backend-protocol-version":"GRPC","projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin | -| flyteadmin.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Flyteadmin pods | +| flyteadmin.serviceAccount | object | `{"annotations":{},"create":true}` | Configuration for service accounts for FlyteAdmin | +| flyteadmin.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Flyteadmin pods | +| flyteadmin.serviceAccount.create | bool | `true` | Should a service account be created for flyteadmin | | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment | | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | | @@ -120,7 +127,9 @@ A Helm chart for Flyte | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | | flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytepropeller deployment | -| flytepropeller.serviceAccountAnnotations | object | `{}` | Annotations for ServiceAccount attached to Flytepropeller pods | +| flytepropeller.serviceAccount | object | `{"annotations":{},"create":true}` | Configuration for service accounts for FlytePropeller | +| flytepropeller.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to FlytePropeller pods | +| flytepropeller.serviceAccount.create | bool | `true` | Should a service account be created for FlytePropeller | | flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment | | kubernetes-dashboard.enabled | bool | `false` | | | minio.affinity | object | `{}` | affinity for Minio deployment | @@ -168,6 +177,9 @@ A Helm chart for Flyte | redis.resources | object | `{"limits":{"cpu":"1000m","memory":"1Gi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Redis Statefulset | | redis.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Redis | | redis.tolerations | list | `[]` | tolerations for Redis Statefulset | +| sagemaker.enabled | bool | `false` | | +| sagemaker.plugin_config.plugins.sagemaker.region | string | `""` | | +| sagemaker.plugin_config.plugins.sagemaker.roleArn | string | `""` | | | sparkoperator.enabled | bool | `true` | | | sparkoperator.image.tag | string | `"v1beta2-1.2.0-3.0.0"` | Docker image for Sparkoperator | | sparkoperator.plugin_config | object | `{"plugins":{"spark":{"spark-config-default":[{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]}}}` | Spark plugin configuration | @@ -179,6 +191,7 @@ A Helm chart for Flyte | storage.gcs | string | `nil` | settings for storage type gcs | | storage.s3 | object | `{"region":"us-east-1"}` | settings for storage type s3 | | storage.type | string | `"sandbox"` | Sets the storage type. Supported values are sandbox, s3, gcs and custom. | +| tf_operator.enabled | bool | `false` | | | workflow_notifications | object | `{"config":{},"enabled":false}` | **Optional Component** Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of various events in their workflows | | workflow_scheduler.config | object | `{}` | | | workflow_scheduler.enabled | bool | `false` | | diff --git a/helm/templates/admin/rbac.yaml b/helm/templates/admin/rbac.yaml index 9696437360..0ccf24f4d1 100644 --- a/helm/templates/admin/rbac.yaml +++ b/helm/templates/admin/rbac.yaml @@ -1,14 +1,16 @@ --- +{{- if .Values.flyteadmin.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: name: {{ template "flyteadmin.name" . }} namespace: {{ template "flyte.namespace" . }} labels: {{ include "flyteadmin.labels" . | nindent 4 }} - {{- with .Values.flyteadmin.serviceAccountAnnotations }} + {{- with .Values.flyteadmin.serviceAccount.annotations }} annotations: {{ toYaml . | nindent 4 }} {{- end}} +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/helm/templates/admin/service.yaml b/helm/templates/admin/service.yaml index 78b5acba81..07132a45d4 100644 --- a/helm/templates/admin/service.yaml +++ b/helm/templates/admin/service.yaml @@ -16,16 +16,19 @@ spec: {{ . }} {{- end }} ports: - - name: redoc - port: 87 - protocol: TCP - targetPort: 8087 - - name: http - port: 80 - protocol: TCP - targetPort: 8088 - - name: grpc - port: 81 - protocol: TCP - targetPort: 8089 + - name: http + port: 80 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 81 + protocol: TCP + targetPort: 8089 + - name: redoc + protocol: TCP + port: 87 + targetPort: 8087 + - name: http-metrics + protocol: TCP + port: 10254 selector: {{ include "flyteadmin.selectorLabels" . | nindent 4 }} diff --git a/helm/templates/datacatalog/rbac.yaml b/helm/templates/datacatalog/rbac.yaml index ca1ab89afd..3c549e31e5 100644 --- a/helm/templates/datacatalog/rbac.yaml +++ b/helm/templates/datacatalog/rbac.yaml @@ -1,5 +1,6 @@ --- +{{- if .Values.datacatalog.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: @@ -9,3 +10,4 @@ metadata: {{- with .Values.datacatalog.serviceAccountAnnotations }} annotations: {{ toYaml . | nindent 4 }} {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/templates/propeller/rbac.yaml b/helm/templates/propeller/rbac.yaml index dd0cee2788..2bc5d7cb61 100644 --- a/helm/templates/propeller/rbac.yaml +++ b/helm/templates/propeller/rbac.yaml @@ -1,5 +1,6 @@ --- +{{- if .Values.flytepropeller.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: @@ -9,7 +10,7 @@ metadata: {{- with .Values.flytepropeller.serviceAccountAnnotations }} annotations: {{ toYaml . | nindent 4 }} {{- end}} - + {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index fb22db05fe..89cafde3e3 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -6,8 +6,11 @@ flyteadmin: replicaCount: 2 # -- IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html - serviceAccountAnnotations: - eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte + serviceAccount: + # -- If the service account is created by you, make this false + create: false + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte resources: limits: @@ -20,14 +23,12 @@ flyteadmin: service: annotations: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" - # TODO add security groups - service.beta.kubernetes.io/aws-load-balancer-extra-security-groups: "sg-...,sg-...,sg-..." # TODO alter domain - external-dns.alpha.kubernetes.io/hostname: "flyteadmin.subdomain.mydomain.com" + external-dns.alpha.kubernetes.io/hostname: "demo2.nuclyde.io" type: LoadBalancer loadBalancerSourceRanges: # TODO change source ip range if desired - - 0.0.0.0 + - 0.0.0.0/0 affinity: podAntiAffinity: @@ -43,9 +44,11 @@ flyteadmin: datacatalog: replicaCount: 2 - # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html - serviceAccountAnnotations: - eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte + serviceAccount: + # -- If the service account is created by you, make this false + create: false + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte resources: limits: cpu: 500m @@ -56,8 +59,13 @@ datacatalog: memory: 200Mi service: annotations: - cloud.google.com/load-balancer-type: Internal + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" + # TODO alter domain + # external-dns.alpha.kubernetes.io/hostname: "demo2.nuclyde.io" type: LoadBalancer + loadBalancerSourceRanges: + # TODO change source ip range if desired + - 0.0.0.0/0 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -72,19 +80,21 @@ datacatalog: flytepropeller: replicaCount: 2 - # IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html - serviceAccountAnnotations: - eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte + serviceAccount: + # -- If the service account is created by you, make this false + create: false + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte resources: limits: cpu: 1 - ephemeral-storage: 1GB - memory: 2GB + ephemeral-storage: 1Gi + memory: 2Gi requests: cpu: 1 - ephemeral-storage: 1GB - memory: 2GB - cacheSizeMbs: 512 + ephemeral-storage: 1Gi + memory: 2Gi + cacheSizeMbs: 1024 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -102,6 +112,15 @@ flyteconsole: resources: limits: cpu: 250m + service: + annotations: + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" + # TODO alter domain + external-dns.alpha.kubernetes.io/hostname: "demo2.nuclyde.io" + type: LoadBalancer + loadBalancerSourceRanges: + # TODO change source ip range if desired + - 0.0.0.0/0 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -165,26 +184,18 @@ common: databaseSecret: name: db-pass secretManifest: - # # Leave it empty if your secret already exists - - # # Else you can create your own secret object: - # # Necessary dependencies: - # # - https://github.com/hashicorp/vault - # # - https://github.com/godaddy/kubernetes-external-secrets - - # apiVersion: kubernetes-client.io/v1 - # kind: ExternalSecret - # metadata: - # name: db-pass - # namespace: flyte - # spec: - # backendType: vault - # vaultMountPoint: - # vaultRole: - # data: - # - name: pass.txt - # key: k8s/data/path/to/secret - # property: password + # -- Leave it empty if your secret already exists + # Else you can create your own secret object. You can use Kubernetes secrets, else you can configure external secrets + # For external secrets please install Necessary dependencies, like, of your choice + # - https://github.com/hashicorp/vault + # - https://github.com/godaddy/kubernetes-external-secrets + apiVersion: v1 + kind: Secret + metadata: + name: db-pass + type: Opaque + stringData: + pass.txt: awesomesauce # ----------------------------------------------------- @@ -207,7 +218,7 @@ db: database: port: 5432 username: flyte - host: flyteadmin.cluster-cguibupmpitq.us-east-2.rds.amazonaws.com + host: flyte-helm-test.cluster-cguibupmpitq.us-east-2.rds.amazonaws.com dbname: flyteadmin passwordPath: /etc/db/pass.txt @@ -273,19 +284,19 @@ configmap: - spark - k8s-array - pytorch - - sagemaker_hyperparameter_tuning - - sagemaker_custom_training - - sagemaker_training - athena +# - sagemaker_hyperparameter_tuning +# - sagemaker_custom_training +# - sagemaker_training default-for-task-types: container: container sidecar: sidecar spark: spark container_array: k8s-array pytorch: pytorch - sagemaker_custom_training_task: sagemaker_custom_training - sagemaker_custom_training_job_task: sagemaker_custom_training hive: athena +# sagemaker_custom_training_task: sagemaker_custom_training +# sagemaker_custom_training_job_task: sagemaker_custom_training logger: logger: @@ -312,24 +323,28 @@ configmap: # All these features are optional, but are critical to run certain features # ------------------------------------------------------------------------ -# -- -# Flyte uses a cloud hosted Cron scheduler to run workflows on a schedule. The following module is optional. Without, +# -- # Flyte uses a cloud hosted Cron scheduler to run workflows on a schedule. The following module is optional. Without, # this module, you will not have scheduled launchplans/workflows. workflow_scheduler: - enabled: true + enabled: false config: scheduler: + # -- This is configured to use Cloudwatch schedules as explained [here](https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/Create-CloudWatch-Events-Scheduled-Rule.html) eventScheduler: scheme: aws - region: us-east-1 - scheduleRole: arn:aws:iam:::role/flyte_cron_scheduler_role - targetName: arn:aws:sqs:us-east-1::flyte-cron-scheduler-queue + region: us-east-2 + # -- This is the role used by cloudwatch to insert a new event into the queue + scheduleRole: arn:aws:iam:::role/flyte_cron_scheduler_role + # -- This is the queue into which cloudwatch will enqueue a request and flyteadmin will consume from. + # This queue should be accessible by flyteadmin + targetName: arn:aws:sqs:us-east-2:: scheduleNamePrefix: flyte workflowExecutor: scheme: aws - region: us-east-1 - scheduleQueueName: flyte-cron-scheduler-queue - accountId: + region: us-east-2 + # -- this is the same queue as eventScheduler.targetName + scheduleQueueName: + accountId: reconnectAttempts: 10 reconnectDelaySeconds: 30 @@ -338,16 +353,19 @@ workflow_scheduler: # Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of # various events in their workflows workflow_notifications: - enabled: true + enabled: false config: notifications: type: aws - region: us-east-1 + region: us-east-2 publisher: - topicName: arn:aws:sns:us-east-1::flyte-notifications-topic + # -- The notifications topic on which event notifications will be sent + topicName: arn:aws:sns:us-east-1:: processor: - queueName: flyte-notifications-queue + # -- SQS queue from which the event notifications will be consumed + queueName: accountId: + # -- For every notification this will be the message sent by FlyteAdmin emailer: subject: "Flyte: {{ project }}/{{ domain }}/{{ launch_plan.name }} has '{{ phase }}'" sender: "flyte@example.com" @@ -494,14 +512,14 @@ pytorchoperator: # -- Distributed tensorflow training using the TF-Operator. This can be installed as explained [here](https://github.com/kubeflow/tf-operator) # There is no helm chart maintained and hence it might need an independent installation tf_operator: - enabled: true + enabled: false # ----------------- # -- Training on AWS Sagemaker using AWS Sagemaker operator. To actually install the operator, please follow instructions [here](https://github.com/aws/amazon-sagemaker-operator-for-k8s/tree/master/hack/charts/installer/rolebased) # Use the config section here to just enable sagemaker plugin in Flyte, after you have installed the operator using the information sagemaker: - enabled: true + enabled: false plugin_config: plugins: sagemaker: diff --git a/helm/values.yaml b/helm/values.yaml index 2c45dd1225..d1ef96eaeb 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -37,8 +37,12 @@ flyteadmin: alb.ingress.kubernetes.io/backend-protocol-version: GRPC type: ClusterIP loadBalancerSourceRanges: [] - # -- Annotations for ServiceAccount attached to Flyteadmin pods - serviceAccountAnnotations: {} + # -- Configuration for service accounts for FlyteAdmin + serviceAccount: + # -- Should a service account be created for flyteadmin + create: true + # -- Annotations for ServiceAccount attached to Flyteadmin pods + annotations: {} # -- Annotations for Flyteadmin pods podAnnotations: {} # -- nodeSelector for Flyteadmin deployment @@ -78,8 +82,12 @@ datacatalog: projectcontour.io/upstream-protocol.h2c: grpc alb.ingress.kubernetes.io/backend-protocol-version: GRPC type: NodePort - # -- Annotations for ServiceAccount attached to Datacatalog pods - serviceAccountAnnotations: {} + # -- Configuration for service accounts for Datacatalog + serviceAccount: + # -- Should a service account be created for Datacatalog + create: true + # -- Annotations for ServiceAccount attached to Datacatalog pods + annotations: {} # -- Annotations for Datacatalog pods podAnnotations: {} # -- nodeSelector for Datacatalog deployment @@ -114,8 +122,14 @@ flytepropeller: cacheSizeMbs: 0 # -- Default regex string for searching configuration files configPath: /etc/flyte/config/*.yaml - # -- Annotations for ServiceAccount attached to Flytepropeller pods - serviceAccountAnnotations: {} + + # -- Configuration for service accounts for FlytePropeller + serviceAccount: + # -- Should a service account be created for FlytePropeller + create: true + # -- Annotations for ServiceAccount attached to FlytePropeller pods + annotations: {} + # -- Annotations for Flytepropeller pods podAnnotations: {} # -- nodeSelector for Flytepropeller deployment @@ -396,6 +410,7 @@ configmap: console: BASE_URL: /console CONFIG_DIR: /etc/flyte/config + DISABLE_AUTH: "1" # -- Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. domain: @@ -588,6 +603,7 @@ configmap: # -- **Optional Component** # Flyte uses a cloud hosted Cron scheduler to run workflows on a schedule. The following module is optional. Without, # this module, you will not have scheduled launchplans / workflows. +# Docs: https://docs.flyte.org/en/latest/howto/enable_and_use_schedules.html#setting-up-scheduled-workflows workflow_scheduler: enabled: false config: {} From 35d1790c1312b50b0fb2955d7713880ca1f8e2de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Tue, 27 Apr 2021 11:40:54 +0200 Subject: [PATCH 32/92] Improve installation docs and bump versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/{README.MD => README.md} | 39 ++++++++++++++++++++++++++++------- helm/README.md.gotmpl | 28 +++++++++++++++++++++++-- helm/values.yaml | 10 ++++----- 3 files changed, 62 insertions(+), 15 deletions(-) rename helm/{README.MD => README.md} (95%) diff --git a/helm/README.MD b/helm/README.md similarity index 95% rename from helm/README.MD rename to helm/README.md index 2c1e8c9fb4..220cfd0e63 100644 --- a/helm/README.MD +++ b/helm/README.md @@ -13,9 +13,32 @@ A Helm chart for Flyte | https://kubernetes.github.io/dashboard/ | kubernetes-dashboard | 4.0.2 | ### SANDBOX INSTALLATION: -- Install `helm 3` realese version -- Generate `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` +- [Install helm 3](https://helm.sh/docs/intro/install/) +- Fetch chart dependencies `helm dep up` +- Install Flyte sandbox: + +```bash +cd helm +helm install -n flyte -f values-sandbox.yaml --create-namespace flyte . +``` + +Customize your installation by changing settings in `values-sandbox.yaml`. +You can use the helm diff plugin to review any value changes you've made to your values: + +```bash +helm plugin install https://github.com/databus23/helm-diff +helm diff upgrade -f values-sandbox.yaml flyte . +``` + +Then apply your changes: +```bash +helm upgrade -f values-sandbox.yaml flyte . +``` + +#### Alternative: Generate raw kubernetes yaml with helm template +- `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` - Deploy the manifest `kubectl apply -f flyte_generated_sandbox.yaml` + - When all pods are running - run end2end tests: `kubectl apply -f ../end2end/tests/endtoend.yaml` - Get flyte host `minikube service contour -n heptio-contour --url`. And then visit `http:///console` @@ -30,7 +53,7 @@ A Helm chart for Flyte |-----|------|---------|-------------| | cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"},{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | | cluster_resource_manager.config | object | `{"cluster_resources":{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}}` | Configmap for ClusterResource parameters | -| cluster_resource_manager.config.cluster_resources | object | `{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters | +| cluster_resource_manager.config.cluster_resources | object | `{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. | | cluster_resource_manager.enabled | bool | `true` | Enables the Cluster resource manager component | | cluster_resource_manager.templates | list | `[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"},{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}]` | Resource templates that should be applied | | cluster_resource_manager.templates[0] | object | `{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"}` | Template for namespaces resources | @@ -67,7 +90,7 @@ A Helm chart for Flyte | configmap.task_logs.plugins.logs.cloudwatch-enabled | bool | `true` | One option is to enable cloudwatch logging for EKS, update the region and log group accordingly | | configmap.task_logs.plugins.logs.cloudwatch-log-group | string | `"/aws/containerinsights/flyte-demo-2/application"` | cloudwatch log-group | | configmap.task_logs.plugins.logs.cloudwatch-region | string | `"us-east-2"` | region where logs are hosted | -| configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}}` | Task default resources configuration | +| configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}}` | Task default resources configuration Refer to the full [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#TaskResourceConfiguration). | | configmap.task_resource_defaults.task_resources | object | `{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}` | Task default resources parameters | | contour.affinity | object | `{}` | affinity for Contour deployment | | contour.contour.resources | object | `{"limits":{"cpu":"100m","memory":"100Mi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Contour | @@ -82,7 +105,7 @@ A Helm chart for Flyte | datacatalog.configPath | string | `"/etc/datacatalog/config/*.yaml"` | Default regex string for searching configuration files | | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | | | datacatalog.image.repository | string | `"ghcr.io/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| datacatalog.image.tag | string | `"v0.3.0"` | | +| datacatalog.image.tag | string | `"v0.3.2"` | | | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | @@ -97,7 +120,7 @@ A Helm chart for Flyte | flyteadmin.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | | | flyteadmin.image.repository | string | `"ghcr.io/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyteadmin.image.tag | string | `"v0.4.0"` | | +| flyteadmin.image.tag | string | `"v0.4.14"` | | | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | | flyteadmin.replicaCount | int | `1` | Replicas count for Flyteadmin deployment | @@ -110,7 +133,7 @@ A Helm chart for Flyte | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | | | flyteconsole.image.repository | string | `"ghcr.io/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyteconsole.image.tag | string | `"v0.19.6"` | | +| flyteconsole.image.tag | string | `"v0.19.7"` | | | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | | flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment | @@ -122,7 +145,7 @@ A Helm chart for Flyte | flytepropeller.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | | | flytepropeller.image.repository | string | `"ghcr.io/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flytepropeller.image.tag | string | `"v0.7.8"` | | +| flytepropeller.image.tag | string | `"v0.10.4"` | | | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | diff --git a/helm/README.md.gotmpl b/helm/README.md.gotmpl index eb4f8c63d5..abbdb920b1 100644 --- a/helm/README.md.gotmpl +++ b/helm/README.md.gotmpl @@ -14,9 +14,33 @@ {{ template "chart.requirementsSection" . }} ### SANDBOX INSTALLATION: -- Install `helm 3` realese version -- Generate `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` +- [Install helm 3](https://helm.sh/docs/intro/install/) +- Fetch chart dependencies `helm dep up` +- Install Flyte sandbox: + +```bash +cd helm +helm install -n flyte -f values-sandbox.yaml --create-namespace flyte . +``` + +Customize your installation by changing settings in `values-sandbox.yaml`. +You can use the helm diff plugin to review any value changes you've made to your values: + +```bash +helm plugin install https://github.com/databus23/helm-diff +helm diff upgrade -f values-sandbox.yaml flyte . +``` + +Then apply your changes: +```bash +helm upgrade -f values-sandbox.yaml flyte . +``` + +#### Alternative: Generate raw kubernetes yaml with helm template +- `helm template --name-template=flyte-sandbox . -n flyte -f values-sandbox.yaml > flyte_generated_sandbox.yaml` - Deploy the manifest `kubectl apply -f flyte_generated_sandbox.yaml` + + - When all pods are running - run end2end tests: `kubectl apply -f ../end2end/tests/endtoend.yaml` - Get flyte host `minikube service contour -n heptio-contour --url`. And then visit `http:///console` diff --git a/helm/values.yaml b/helm/values.yaml index d1ef96eaeb..f78258902b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -16,7 +16,7 @@ flyteadmin: image: # -- Docker image for Flyteadmin deployment repository: ghcr.io/flyteorg/flyteadmin - tag: v0.4.0 + tag: v0.4.14 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteadmin deployment resources: @@ -62,7 +62,7 @@ datacatalog: image: # -- Docker image for Datacatalog deployment repository: ghcr.io/flyteorg/datacatalog - tag: v0.3.0 + tag: v0.3.2 pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment resources: @@ -107,7 +107,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: ghcr.io/flyteorg/flytepropeller - tag: v0.7.8 + tag: v0.10.4 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -149,7 +149,7 @@ flyteconsole: image: # -- Docker image for Flyteconsole deployment repository: ghcr.io/flyteorg/flyteconsole - tag: v0.19.6 + tag: v0.19.7 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment resources: @@ -395,7 +395,7 @@ db: # host: postgres # dbname: flyte # passwordPath: /etc/db/pass.txt - + # options: # -------------------------------------------------------------------- From 919a2a7181fe5293aa2a31d5e0bf23cd7c238b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Tue, 27 Apr 2021 11:43:01 +0200 Subject: [PATCH 33/92] Fix service account annotation structure in templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/common/ingress.yaml | 4 ++-- helm/templates/datacatalog/rbac.yaml | 2 +- helm/templates/propeller/rbac.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index d1aaa3b114..3b9091ce29 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -181,7 +181,7 @@ spec: {{- end }} {{- if .Values.common.ingress.tls.enabled }} tls: - - secretName: {{ .Release.Name }}-tls + - secretName: {{ .Release.Name }}-flyte-tls hosts: - {{ .Values.common.ingress.host }} {{ end }} @@ -234,7 +234,7 @@ spec: {{- end }} {{- if .Values.common.ingress.tls.enabled }} tls: - - secretName: {{ .Release.Name }}-tls + - secretName: {{ .Release.Name }}-flyte-tls hosts: - {{ .Values.common.ingress.host }} {{ end }} diff --git a/helm/templates/datacatalog/rbac.yaml b/helm/templates/datacatalog/rbac.yaml index 3c549e31e5..1813bce1af 100644 --- a/helm/templates/datacatalog/rbac.yaml +++ b/helm/templates/datacatalog/rbac.yaml @@ -7,7 +7,7 @@ metadata: name: {{ template "datacatalog.name" . }} namespace: {{ template "flyte.namespace" . }} labels: {{ include "datacatalog.labels" . | nindent 4 }} - {{- with .Values.datacatalog.serviceAccountAnnotations }} + {{- with .Values.datacatalog.serviceAccount.annotations }} annotations: {{ toYaml . | nindent 4 }} {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/templates/propeller/rbac.yaml b/helm/templates/propeller/rbac.yaml index 2bc5d7cb61..003704cf3b 100644 --- a/helm/templates/propeller/rbac.yaml +++ b/helm/templates/propeller/rbac.yaml @@ -7,7 +7,7 @@ metadata: name: {{ template "flytepropeller.name" . }} namespace: {{ template "flyte.namespace" . }} labels: {{ include "flytepropeller.labels" . | nindent 4 }} - {{- with .Values.flytepropeller.serviceAccountAnnotations }} + {{- with .Values.flytepropeller.serviceAccount.annotations }} annotations: {{ toYaml . | nindent 4 }} {{- end}} {{- end }} From e2bda959d12129f583626a84afbccb2a102fcf4b Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Mon, 26 Apr 2021 13:20:36 -0700 Subject: [PATCH 34/92] Fully working Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- helm/README.md | 10 ++--- helm/templates/common/ingress.yaml | 4 +- helm/values-eks.yaml | 61 +++++++++++++----------------- helm/values.yaml | 8 +--- 4 files changed, 35 insertions(+), 48 deletions(-) diff --git a/helm/README.md b/helm/README.md index 220cfd0e63..ba32673ebc 100644 --- a/helm/README.md +++ b/helm/README.md @@ -86,10 +86,8 @@ helm upgrade -f values-sandbox.yaml flyte . | configmap.resource_manager | object | `{"propeller":{"resourcemanager":{"redis":{"hostKey":"mypassword","hostPath":"redis-resource-manager:6379"},"resourceMaxQuota":10000,"type":"redis"}}}` | Resource manager configuration | | configmap.resource_manager.propeller | object | `{"resourcemanager":{"redis":{"hostKey":"mypassword","hostPath":"redis-resource-manager:6379"},"resourceMaxQuota":10000,"type":"redis"}}` | resource manager configuration | | configmap.server | object | `{"flyteadmin":{"eventVersion":1,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | -| configmap.task_logs | object | `{"plugins":{"logs":{"cloudwatch-enabled":true,"cloudwatch-log-group":"/aws/containerinsights/flyte-demo-2/application","cloudwatch-region":"us-east-2","kubernetes-enabled":false}}}` | Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various logging engines | -| configmap.task_logs.plugins.logs.cloudwatch-enabled | bool | `true` | One option is to enable cloudwatch logging for EKS, update the region and log group accordingly | -| configmap.task_logs.plugins.logs.cloudwatch-log-group | string | `"/aws/containerinsights/flyte-demo-2/application"` | cloudwatch log-group | -| configmap.task_logs.plugins.logs.cloudwatch-region | string | `"us-east-2"` | region where logs are hosted | +| configmap.task_logs | object | `{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":false}}}` | Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various logging engines | +| configmap.task_logs.plugins.logs.cloudwatch-enabled | bool | `false` | One option is to enable cloudwatch logging for EKS, update the region and log group accordingly | | configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}}` | Task default resources configuration Refer to the full [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#TaskResourceConfiguration). | | configmap.task_resource_defaults.task_resources | object | `{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}` | Task default resources parameters | | contour.affinity | object | `{}` | affinity for Contour deployment | @@ -110,7 +108,7 @@ helm upgrade -f values-sandbox.yaml flyte . | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | | datacatalog.resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Datacatalog deployment | -| datacatalog.service | object | `{"annotations":{"alb.ingress.kubernetes.io/backend-protocol-version":"GRPC","projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"}` | Service settings for Datacatalog | +| datacatalog.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"}` | Service settings for Datacatalog | | datacatalog.serviceAccount | object | `{"annotations":{},"create":true}` | Configuration for service accounts for Datacatalog | | datacatalog.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Datacatalog pods | | datacatalog.serviceAccount.create | bool | `true` | Should a service account be created for Datacatalog | @@ -125,7 +123,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | | flyteadmin.replicaCount | int | `1` | Replicas count for Flyteadmin deployment | | flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment | -| flyteadmin.service | object | `{"annotations":{"alb.ingress.kubernetes.io/backend-protocol-version":"GRPC","projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin | +| flyteadmin.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin | | flyteadmin.serviceAccount | object | `{"annotations":{},"create":true}` | Configuration for service accounts for FlyteAdmin | | flyteadmin.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Flyteadmin pods | | flyteadmin.serviceAccount.create | bool | `true` | Should a service account be created for flyteadmin | diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index 3b9091ce29..86eb76be9b 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -185,6 +185,8 @@ spec: hosts: - {{ .Values.common.ingress.host }} {{ end }} + + {{- if .Values.common.ingress.separateGrpcIngress }} # Certain ingress controllers like nginx cannot serve HTTP 1 and GRPC with a single ingress because GRPC can only # enabled on the ingress object, not on backend services (GRPC annotation is set on the ingress, not on the services). @@ -210,7 +212,7 @@ spec: service: name: ssl-redirect port: - name: ssl-redirect + name: use-annotation path: /* pathType: ImplementationSpecific {{- end }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 89cafde3e3..75c5ef934b 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -7,7 +7,8 @@ flyteadmin: replicaCount: 2 # -- IAM role for SA: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html serviceAccount: - # -- If the service account is created by you, make this false + # -- If the service account is created by you, make this false, else a new service account will be created and the iam-role-flyte will be added + # you can change the name of this role create: false annotations: eks.amazonaws.com/role-arn: arn:aws:iam:::role/iam-role-flyte @@ -24,7 +25,7 @@ flyteadmin: annotations: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" # TODO alter domain - external-dns.alpha.kubernetes.io/hostname: "demo2.nuclyde.io" + external-dns.alpha.kubernetes.io/hostname: "flyte.example.com" type: LoadBalancer loadBalancerSourceRanges: # TODO change source ip range if desired @@ -60,8 +61,6 @@ datacatalog: service: annotations: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" - # TODO alter domain - # external-dns.alpha.kubernetes.io/hostname: "demo2.nuclyde.io" type: LoadBalancer loadBalancerSourceRanges: # TODO change source ip range if desired @@ -116,7 +115,7 @@ flyteconsole: annotations: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" # TODO alter domain - external-dns.alpha.kubernetes.io/hostname: "demo2.nuclyde.io" + external-dns.alpha.kubernetes.io/hostname: "flyte.example.com" type: LoadBalancer loadBalancerSourceRanges: # TODO change source ip range if desired @@ -168,19 +167,20 @@ contour: common: ingress: - host: demo2.nuclyde.io + host: flyte.example.com albSSLRedirect: true + separateGrpcIngress: true annotations: # -- aws-load-balancer-controller v2.1 or higher is required - https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.1/ # For EKS if using [ALB](https://kubernetes-sigs.github.io/aws-load-balancer-controller/guide/ingress/annotations/), these annotations are set kubernetes.io/ingress.class: alb alb.ingress.kubernetes.io/tags: service_instance=production alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:590375264460:certificate/e2f04275-2dff-4118-a493-ed3ec8f41605 + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2::certificate/e2f04275-2dff-4118-a493-ed3ec8f41605 alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' # -- Instruct ALB Controller to not create multiple load balancers (and hence maintain a single endpoint for both GRPC and Http) - alb.ingress.kubernetes.io/group.name: flytesystem + alb.ingress.kubernetes.io/group.name: flyte databaseSecret: name: db-pass secretManifest: @@ -195,6 +195,7 @@ common: name: db-pass type: Opaque stringData: + # -- If using plain text you can provide the password here pass.txt: awesomesauce @@ -210,15 +211,17 @@ storage: # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. type: s3 # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. - bucketName: flyte-demo + bucketName: s3: - region: us-east-2 + region: db: database: port: 5432 + # -- Create a user called flyte username: flyte - host: flyte-helm-test.cluster-cguibupmpitq.us-east-2.rds.amazonaws.com + host: + # -- Create a DB called flyteadmin (OR change the name here) dbname: flyteadmin passwordPath: /etc/db/pass.txt @@ -310,11 +313,12 @@ configmap: logs: kubernetes-enabled: false # -- One option is to enable cloudwatch logging for EKS, update the region and log group accordingly + # You can even disable this cloudwatch-enabled: true # -- region where logs are hosted - cloudwatch-region: us-east-2 + cloudwatch-region: # -- cloudwatch log-group - cloudwatch-log-group: /aws/containerinsights/flyte-demo-2/application + cloudwatch-log-group: # ---------------------------------------------------------------- @@ -333,17 +337,13 @@ workflow_scheduler: eventScheduler: scheme: aws region: us-east-2 - # -- This is the role used by cloudwatch to insert a new event into the queue scheduleRole: arn:aws:iam:::role/flyte_cron_scheduler_role - # -- This is the queue into which cloudwatch will enqueue a request and flyteadmin will consume from. - # This queue should be accessible by flyteadmin - targetName: arn:aws:sqs:us-east-2:: + targetName: arn:aws:sqs:us-east-2::flyte-helm-test-cron-scheduler-queue scheduleNamePrefix: flyte workflowExecutor: scheme: aws region: us-east-2 - # -- this is the same queue as eventScheduler.targetName - scheduleQueueName: + scheduleQueueName: flyte-helm-test-cron-scheduler-queue accountId: reconnectAttempts: 10 reconnectDelaySeconds: 30 @@ -359,13 +359,10 @@ workflow_notifications: type: aws region: us-east-2 publisher: - # -- The notifications topic on which event notifications will be sent - topicName: arn:aws:sns:us-east-1:: + topicName: arn:aws:sns:us-east-1::flyte-notifications-topic processor: - # -- SQS queue from which the event notifications will be consumed - queueName: + queueName: flyte-notifications-queue accountId: - # -- For every notification this will be the message sent by FlyteAdmin emailer: subject: "Flyte: {{ project }}/{{ domain }}/{{ launch_plan.name }} has '{{ phase }}'" sender: "flyte@example.com" @@ -404,7 +401,7 @@ cluster_resource_manager: limits.cpu: {{ projectQuotaCpu }} limits.memory: {{ projectQuotaMemory }} - # -- Template for spark role + # -- Template copilot configuration - key: ac_project_copilot_dataconfig value: | kind: ConfigMap @@ -415,15 +412,11 @@ cluster_resource_manager: data: config.yaml: | storage: + type: s3 + container: connection: - access-key: minio - auth-type: accesskey - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - secret-key: miniostorage - type: minio - container: my-s3-bucket + auth-type: iam + region: enable-multicontainer: true # -- Template for spark role @@ -523,7 +516,7 @@ sagemaker: plugin_config: plugins: sagemaker: - roleArn: arn:aws:iam::590375264460:role/eksctl-flyte-demo-2-addon-iamserviceaccount-Role1-62EXWRIT27CA + roleArn: arn:aws:iam:::role/ region: us-east-2 diff --git a/helm/values.yaml b/helm/values.yaml index f78258902b..e828ca022f 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -34,7 +34,6 @@ flyteadmin: service: annotations: projectcontour.io/upstream-protocol.h2c: grpc - alb.ingress.kubernetes.io/backend-protocol-version: GRPC type: ClusterIP loadBalancerSourceRanges: [] # -- Configuration for service accounts for FlyteAdmin @@ -80,7 +79,6 @@ datacatalog: service: annotations: projectcontour.io/upstream-protocol.h2c: grpc - alb.ingress.kubernetes.io/backend-protocol-version: GRPC type: NodePort # -- Configuration for service accounts for Datacatalog serviceAccount: @@ -588,11 +586,7 @@ configmap: logs: kubernetes-enabled: false # -- One option is to enable cloudwatch logging for EKS, update the region and log group accordingly - cloudwatch-enabled: true - # -- region where logs are hosted - cloudwatch-region: us-east-2 - # -- cloudwatch log-group - cloudwatch-log-group: /aws/containerinsights/flyte-demo-2/application + cloudwatch-enabled: false # ---------------------------------------------------------------- # Optional Modules From 7dd63fa00e6fd135e8f8c9063b020d7607f1fa50 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Thu, 6 May 2021 09:46:06 -0700 Subject: [PATCH 35/92] Update image tags, fix customDataConfig and expose defaultIamRole as a template field Signed-off-by: Haytham Abuelfutuh --- helm/values-eks.yaml | 72 +++++++++++++++++++++----------------------- helm/values.yaml | 8 ++--- 2 files changed, 39 insertions(+), 41 deletions(-) diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 75c5ef934b..2f270f01a0 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -1,4 +1,3 @@ - # # FLYTEADMIN # @@ -159,8 +158,6 @@ minio: contour: enabled: false - - # # COMMON # @@ -198,7 +195,6 @@ common: # -- If using plain text you can provide the password here pass.txt: awesomesauce - # ----------------------------------------------------- # Core dependencies that should be configured for Flyte to work on any platform # Specifically 2 - Storage (s3, gcs etc), Production RDBMS - Aurora, CloudSQL etc @@ -244,11 +240,6 @@ configmap: allowedHeaders: - "Content-Type" - task_type_whitelist: - spark: - - project: flytetester - - project: spark-workflows - task_resource_defaults: task_resources: defaults: @@ -288,9 +279,9 @@ configmap: - k8s-array - pytorch - athena -# - sagemaker_hyperparameter_tuning -# - sagemaker_custom_training -# - sagemaker_training + # - sagemaker_hyperparameter_tuning + # - sagemaker_custom_training + # - sagemaker_training default-for-task-types: container: container sidecar: sidecar @@ -298,8 +289,8 @@ configmap: container_array: k8s-array pytorch: pytorch hive: athena -# sagemaker_custom_training_task: sagemaker_custom_training -# sagemaker_custom_training_job_task: sagemaker_custom_training + # sagemaker_custom_training_task: sagemaker_custom_training + # sagemaker_custom_training_job_task: sagemaker_custom_training logger: logger: @@ -320,7 +311,6 @@ configmap: # -- cloudwatch log-group cloudwatch-log-group: - # ---------------------------------------------------------------- # Optional Modules # Flyte built extensions that enable various additional features in Flyte. @@ -348,29 +338,27 @@ workflow_scheduler: reconnectAttempts: 10 reconnectDelaySeconds: 30 - # -- # Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of # various events in their workflows workflow_notifications: - enabled: false - config: - notifications: - type: aws - region: us-east-2 - publisher: - topicName: arn:aws:sns:us-east-1::flyte-notifications-topic - processor: - queueName: flyte-notifications-queue - accountId: - emailer: - subject: "Flyte: {{ project }}/{{ domain }}/{{ launch_plan.name }} has '{{ phase }}'" - sender: "flyte@example.com" - body: | - Execution {{ workflow.project }}/{{ workflow.domain }}/{{ workflow.name }}/{{ name }} has {{ phase }}. - Details: https://flyte.example.com/console/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. - {{ error }} - + enabled: false + config: + notifications: + type: aws + region: us-east-2 + publisher: + topicName: arn:aws:sns:us-east-1::flyte-notifications-topic + processor: + queueName: flyte-notifications-queue + accountId: + emailer: + subject: "Flyte: {{ project }}/{{ domain }}/{{ launch_plan.name }} has '{{ phase }}'" + sender: "flyte@example.com" + body: | + Execution {{ workflow.project }}/{{ workflow.domain }}/{{ workflow.name }}/{{ name }} has {{ phase }}. + Details: https://flyte.example.com/console/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. + {{ error }} # -- Configuration for the Cluster resource manager component. This is an optional component, that enables automatic # cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain @@ -389,6 +377,17 @@ cluster_resource_manager: finalizers: - kubernetes + # -- Patch default service account + - key: aab_default_service_account + value: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + namespace: {{ namespace }} + annotations: + eks.amazonaws.com/role-arn: {{ defaultIamRole }} + - key: ab_project_resource_quota value: | apiVersion: v1 @@ -455,6 +454,8 @@ cluster_resource_manager: metadata: name: spark namespace: {{ namespace }} + annotations: + eks.amazonaws.com/role-arn: {{ defaultIamRole }} # -- Template for spark role binding - key: af_spark_role_binding @@ -507,7 +508,6 @@ pytorchoperator: tf_operator: enabled: false - # ----------------- # -- Training on AWS Sagemaker using AWS Sagemaker operator. To actually install the operator, please follow instructions [here](https://github.com/aws/amazon-sagemaker-operator-for-k8s/tree/master/hack/charts/installer/rolebased) # Use the config section here to just enable sagemaker plugin in Flyte, after you have installed the operator using the information @@ -518,8 +518,6 @@ sagemaker: sagemaker: roleArn: arn:aws:iam:::role/ region: us-east-2 - - # --------------------------- # Flink jobs using the Flink Operator # diff --git a/helm/values.yaml b/helm/values.yaml index e828ca022f..8cfb7e8d40 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -105,7 +105,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: ghcr.io/flyteorg/flytepropeller - tag: v0.10.4 + tag: v0.10.7 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -625,17 +625,17 @@ cluster_resource_manager: cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" customData: - production: + - production: - projectQuotaCpu: value: "5" - projectQuotaMemory: value: "4000Mi" - staging: + - staging: - projectQuotaCpu: value: "2" - projectQuotaMemory: value: "3000Mi" - development: + - development: - projectQuotaCpu: value: "4" - projectQuotaMemory: From 4175f26bfc13e25f6f6b3fdbf04f1d5663cb1649 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Thu, 6 May 2021 12:47:54 -0700 Subject: [PATCH 36/92] Allow remote data config for admin Signed-off-by: Haytham Abuelfutuh --- helm/templates/admin/configmap.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm/templates/admin/configmap.yaml b/helm/templates/admin/configmap.yaml index d1895000fc..d8268a5a2e 100644 --- a/helm/templates/admin/configmap.yaml +++ b/helm/templates/admin/configmap.yaml @@ -16,6 +16,9 @@ data: {{- end }} {{- with .Values.configmap.server }} server.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.configmap.remoteData }} + remoteData.yaml: | {{ toYaml . | nindent 4 }} {{- end }} storage.yaml: | {{ include "storage" . | nindent 4 }} {{- with .Values.configmap.task_resource_defaults }} From 3346a5a86f0f762502bb4bef32f0786c7786d0db Mon Sep 17 00:00:00 2001 From: Juan Diego Palomino Date: Thu, 6 May 2021 14:35:04 -0700 Subject: [PATCH 37/92] Use bet version of ingress (#988) "Hey so, I noticed that you are using the Ingress class out of beta, which is kind of an issue-- the Ingress class was moved out of beta in the 1.19 release, which happened in late august. If flyte is run in clusters older than this (e.g. 1.18), then this will fail. EKS only started offering 1.19 a couple of months ago, and the GKE regular channel defaults to 1.18, so this really won't work for most of your users. Do you guys mind if I file a pr to move back to the beta ingress version? I don't see you guys using anyhting required from the GA version, and beta has been serving everyone well for the past 6 years" Signed-off-by: JD Signed-off-by: Haytham Abuelfutuh --- helm/templates/common/ingress.yaml | 154 ++++++++++------------------- 1 file changed, 52 insertions(+), 102 deletions(-) diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index 86eb76be9b..22c4b33d6b 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -1,5 +1,5 @@ {{- if .Values.common.ingress.enabled }} -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ template "flyte.name" . }} @@ -15,166 +15,122 @@ spec: - path: /* pathType: ImplementationSpecific backend: - service: - name: ssl-redirect - port: - name: use-annotation + serviceName: ssl-redirect + servicePort: use-annotation {{- end }} # This is useful only for frontend development {{- if .Values.common.ingress.webpackHMR }} - path: /__webpack_hmr pathType: ImplementationSpecific backend: - service: - name: flyteconsole - port: - number: 80 + serviceName: flyteconsole + servicePort: 80 {{- end }} # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 87 + serviceName: flyteadmin + servicePort: 87 # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml - path: /console pathType: ImplementationSpecific backend: - service: - name: flyteconsole - port: - number: 80 + serviceName: flyteconsole + servicePort: 80 - path: /console/* pathType: ImplementationSpecific backend: - service: - name: flyteconsole - port: - number: 80 + serviceName: flyteconsole + servicePort: 80 - path: /api pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /api/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /healthcheck pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /v1/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /.well-known/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /login pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /login/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /logout pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /logout/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /callback pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /callback/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /me pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /config pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 - path: /config/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 80 + serviceName: flyteadmin + servicePort: 80 {{- if not .Values.common.ingress.separateGrpcIngress }} # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. - path: /flyteidl.service.AdminService pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 81 + serviceName: flyteadmin + servicePort: 81 - path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 81 + serviceName: flyteadmin + servicePort: 81 {{- end }} {{- with .Values.common.ingress.host }} host: {{ . }} @@ -191,7 +147,7 @@ spec: # Certain ingress controllers like nginx cannot serve HTTP 1 and GRPC with a single ingress because GRPC can only # enabled on the ingress object, not on backend services (GRPC annotation is set on the ingress, not on the services). --- -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ template "flyte.name" . }}-grpc @@ -209,10 +165,8 @@ spec: paths: {{- if .Values.common.ingress.albSSLRedirect }} - backend: - service: - name: ssl-redirect - port: - name: use-annotation + serviceName: ssl-redirect + servicePort: use-annotation path: /* pathType: ImplementationSpecific {{- end }} @@ -220,17 +174,13 @@ spec: - path: /flyteidl.service.AdminService pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 81 + serviceName: flyteadmin + servicePort: 81 - path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific backend: - service: - name: flyteadmin - port: - number: 81 + serviceName: flyteadmin + servicePort: 81 {{- with .Values.common.ingress.host }} host: {{ . }} {{- end }} From 54c145cb445ac31979024dabcdc63f4ddb8a035a Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Thu, 6 May 2021 14:38:22 -0700 Subject: [PATCH 38/92] Add auth services to ingress Signed-off-by: Haytham Abuelfutuh --- helm/templates/common/ingress.yaml | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index 22c4b33d6b..3fdaa05ca6 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -131,6 +131,26 @@ spec: backend: serviceName: flyteadmin servicePort: 81 + - path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 {{- end }} {{- with .Values.common.ingress.host }} host: {{ . }} @@ -181,6 +201,26 @@ spec: backend: serviceName: flyteadmin servicePort: 81 + - path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 {{- with .Values.common.ingress.host }} host: {{ . }} {{- end }} From 9c17cebdf8b158204b1dfd671a6062db8ed261ca Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Fri, 7 May 2021 10:36:24 -0700 Subject: [PATCH 39/92] Support imagePullSecrets for service accounts Signed-off-by: Haytham Abuelfutuh --- helm/templates/admin/rbac.yaml | 3 +++ helm/templates/datacatalog/rbac.yaml | 3 +++ helm/templates/propeller/rbac.yaml | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/helm/templates/admin/rbac.yaml b/helm/templates/admin/rbac.yaml index 0ccf24f4d1..c14d8ceba3 100644 --- a/helm/templates/admin/rbac.yaml +++ b/helm/templates/admin/rbac.yaml @@ -10,6 +10,9 @@ metadata: {{- with .Values.flyteadmin.serviceAccount.annotations }} annotations: {{ toYaml . | nindent 4 }} {{- end}} +{{- with .Values.flyteadmin.serviceAccount.imagePullSecrets }} +imagePullSecrets: {{ toYaml . | nindent 4 }} +{{- end}} {{- end }} --- diff --git a/helm/templates/datacatalog/rbac.yaml b/helm/templates/datacatalog/rbac.yaml index 1813bce1af..24f95dc355 100644 --- a/helm/templates/datacatalog/rbac.yaml +++ b/helm/templates/datacatalog/rbac.yaml @@ -10,4 +10,7 @@ metadata: {{- with .Values.datacatalog.serviceAccount.annotations }} annotations: {{ toYaml . | nindent 4 }} {{- end }} +{{- with .Values.datacatalog.serviceAccount.imagePullSecrets }} +imagePullSecrets: {{ toYaml . | nindent 4 }} +{{- end}} {{- end }} \ No newline at end of file diff --git a/helm/templates/propeller/rbac.yaml b/helm/templates/propeller/rbac.yaml index 003704cf3b..768653fc90 100644 --- a/helm/templates/propeller/rbac.yaml +++ b/helm/templates/propeller/rbac.yaml @@ -10,7 +10,10 @@ metadata: {{- with .Values.flytepropeller.serviceAccount.annotations }} annotations: {{ toYaml . | nindent 4 }} {{- end}} - {{- end }} +{{- with .Values.flytepropeller.serviceAccount.imagePullSecrets }} +imagePullSecrets: {{ toYaml . | nindent 4 }} +{{- end}} +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole From e89b23bed2d541ca513e7b2c8d3b38886a2d2d5d Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Mon, 10 May 2021 10:57:41 -0700 Subject: [PATCH 40/92] Update Console version Signed-off-by: Haytham Abuelfutuh --- helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/values.yaml b/helm/values.yaml index 8cfb7e8d40..6b041d1a1e 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -147,7 +147,7 @@ flyteconsole: image: # -- Docker image for Flyteconsole deployment repository: ghcr.io/flyteorg/flyteconsole - tag: v0.19.7 + tag: v0.20.0 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment resources: From 889cd666a68ecdb1289c8f8c263466763e56ab67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Fri, 7 May 2021 11:19:00 +0200 Subject: [PATCH 41/92] Add Auth support to Helm chart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move grpc routes into template function Based on - 840f250d Add Auth Ingress endpoints (#982) - 1ede3750 Authentication Config Examples (#970) - 77941a32 explicitly grant flyteworkflow finalizer permissions to flytepropeller (#930) Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/admin/deployment.yaml | 143 +++++++++++++---------- helm/templates/common/ingress.yaml | 102 +++++++--------- helm/templates/propeller/deployment.yaml | 9 +- helm/templates/propeller/rbac.yaml | 6 + helm/values-gcp.yaml | 20 ++++ helm/values.yaml | 46 +++++++- 6 files changed, 200 insertions(+), 126 deletions(-) diff --git a/helm/templates/admin/deployment.yaml b/helm/templates/admin/deployment.yaml index d088d44c6e..da644f9731 100644 --- a/helm/templates/admin/deployment.yaml +++ b/helm/templates/admin/deployment.yaml @@ -18,57 +18,73 @@ spec: labels: {{ include "flyteadmin.labels" . | nindent 8 }} spec: initContainers: - {{- if .Values.postgres.enabled }} - - name: check-db-ready - image: postgres:10.16-alpine - command: - - sh - - -c - - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done; - {{- end }} - - command: - - flyteadmin - - --config - - {{ .Values.flyteadmin.configPath }} - - migrate - - run - image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" - imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" - name: run-migrations - volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} - - mountPath: /etc/flyte/config - name: config-volume - - command: - - flyteadmin - - --config - - {{ .Values.flyteadmin.configPath }} - - migrate - - seed-projects - - flytesnacks - - flytetester - - flyteexamples - image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" - imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" - name: seed-projects - volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} - - mountPath: /etc/flyte/config - name: config-volume - {{- if .Values.cluster_resource_manager.enabled }} - - command: - - flyteadmin - - --config - - {{ .Values.flyteadmin.configPath }} - - clusterresource - - sync - image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" - imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" - name: sync-cluster-resources - volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} - - mountPath: /etc/flyte/clusterresource/templates - name: resource-templates - - mountPath: /etc/flyte/config - name: config-volume - {{- end }} + {{- if .Values.postgres.enabled }} + - name: check-db-ready + image: postgres:10.16-alpine + command: + - sh + - -c + - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done; + {{- end }} + - command: + - flyteadmin + - --config + - {{ .Values.flyteadmin.configPath }} + - migrate + - run + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + name: run-migrations + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --config + - {{ .Values.flyteadmin.configPath }} + - migrate + - seed-projects + - flytesnacks + - flytetester + - flyteexamples + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + name: seed-projects + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/flyte/config + name: config-volume + {{- if .Values.cluster_resource_manager.enabled }} + - command: + - flyteadmin + - --config + - {{ .Values.flyteadmin.configPath }} + - clusterresource + - sync + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + name: sync-cluster-resources + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + {{- end }} + - name: generate-secrets + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + command: ["/bin/sh", "-c"] + args: + [ + "flyteadmin --config={{ .Values.flyteadmin.configPath }} secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace containers: - command: - flyteadmin @@ -90,6 +106,8 @@ spec: name: shared-data - mountPath: /etc/flyte/config name: config-volume + - name: auth + mountPath: /etc/secrets/ - command: - sh - -c @@ -112,16 +130,19 @@ spec: memory: 200Mi serviceAccountName: {{ template "flyteadmin.name" . }} volumes: {{- include "databaseSecret.volume" . | nindent 6 }} - - emptyDir: {} - name: shared-data - - configMap: - name: flyte-admin-config - name: config-volume - {{- if .Values.cluster_resource_manager.enabled }} - - configMap: - name: clusterresource-template - name: resource-templates - {{- end }} + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-admin-config + name: config-volume + {{- if .Values.cluster_resource_manager.enabled }} + - configMap: + name: clusterresource-template + name: resource-templates + {{- end }} + - name: auth + secret: + secretName: flyte-admin-auth {{- with .Values.flyteadmin.nodeSelector }} nodeSelector: {{ toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index 3fdaa05ca6..7b70e4c5d5 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -1,3 +1,36 @@ +{{- define "grpcRoutes" -}} +# NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. +- path: /flyteidl.service.AdminService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 +- path: /flyteidl.service.AdminService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 +- path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 +- path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 +- path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 +- path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 +{{- end }} {{- if .Values.common.ingress.enabled }} apiVersion: networking.k8s.io/v1beta1 kind: Ingress @@ -69,6 +102,11 @@ spec: backend: serviceName: flyteadmin servicePort: 80 + - path: /.well-known + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 - path: /.well-known/* pathType: ImplementationSpecific backend: @@ -120,37 +158,7 @@ spec: serviceName: flyteadmin servicePort: 80 {{- if not .Values.common.ingress.separateGrpcIngress }} - # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. - - path: /flyteidl.service.AdminService - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.AdminService/* - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.AuthMetadataService - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.AuthMetadataService/* - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.IdentityService - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.IdentityService/* - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 + {{- include "grpcRoutes" . | nindent 10 -}} {{- end }} {{- with .Values.common.ingress.host }} host: {{ . }} @@ -190,37 +198,7 @@ spec: path: /* pathType: ImplementationSpecific {{- end }} - # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. - - path: /flyteidl.service.AdminService - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.AdminService/* - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.AuthMetadataService - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.AuthMetadataService/* - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.IdentityService - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - - path: /flyteidl.service.IdentityService/* - pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 + {{- include "grpcRoutes" . | nindent 10 -}} {{- with .Values.common.ingress.host }} host: {{ . }} {{- end }} diff --git a/helm/templates/propeller/deployment.yaml b/helm/templates/propeller/deployment.yaml index fc11b1eb3d..7b5ae7e63a 100644 --- a/helm/templates/propeller/deployment.yaml +++ b/helm/templates/propeller/deployment.yaml @@ -37,13 +37,18 @@ spec: - containerPort: 10254 resources: {{ toYaml .Values.flytepropeller.resources | nindent 10 }} volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume + - name: config-volume + mountPath: /etc/flyte/config + - name: auth + mountPath: /etc/secrets/ serviceAccountName: {{ template "flytepropeller.name" . }} volumes: - configMap: name: flyte-propeller-config name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth {{- with .Values.flytepropeller.nodeSelector }} nodeSelector: {{ toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/propeller/rbac.yaml b/helm/templates/propeller/rbac.yaml index 768653fc90..b0082fea01 100644 --- a/helm/templates/propeller/rbac.yaml +++ b/helm/templates/propeller/rbac.yaml @@ -21,6 +21,7 @@ metadata: name: {{ template "flytepropeller.name" . }} labels: {{ include "flytepropeller.labels" . | nindent 4 }} rules: +# Allow RO access to PODS - apiGroups: - "" resources: @@ -29,6 +30,7 @@ rules: - get - list - watch +# Allow Event recording access - apiGroups: - "" resources: @@ -38,6 +40,7 @@ rules: - update - delete - patch +# Allow Access All plugin objects - apiGroups: - '*' resources: @@ -50,6 +53,7 @@ rules: - update - delete - patch +# Allow Access to CRD - apiGroups: - apiextensions.k8s.io resources: @@ -61,10 +65,12 @@ rules: - create - delete - update +# Allow Access to all resources under flyte.lyft.com - apiGroups: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list diff --git a/helm/values-gcp.yaml b/helm/values-gcp.yaml index cbe96ceee7..8f9a0069f8 100644 --- a/helm/values-gcp.yaml +++ b/helm/values-gcp.yaml @@ -6,11 +6,31 @@ common: databaseSecret: {} ingress: +# ----------------------------------------------------- +# Core dependencies that should be configured for Flyte to work on any platform +# Specifically 2 - Storage (s3, gcs etc), Production RDBMS - Aurora, CloudSQL etc +# ------------------------------------------------------ +# +# STORAGE SETTINGS +# + +storage: + # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. + type: gcs + # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. + bucketName: + gcs: + projectId: + # # CONFIGMAPS # configmap: + remoteData: + remoteData: + scheme: "gcp" + tasks: max-plugin-phase-versions: 1000000 task-plugins: diff --git a/helm/values.yaml b/helm/values.yaml index 6b041d1a1e..74679b6d78 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -427,9 +427,17 @@ configmap: httpPort: 8088 grpcPort: 8089 security: + # -- Controls whether to serve requests over SSL/TLS. secure: false + # -- Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true + allowedOrigins: + # Accepting all domains for Sandbox installation + - "*" + allowedHeaders: + - "Content-Type" + - "flyte-authorization" # Refer to the full [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ApplicationConfig) for documentation. flyteadmin: roleNameKey: "iam.amazonaws.com/role" @@ -441,7 +449,34 @@ configmap: eventVersion: 1 testing: host: http://flyteadmin - + # -- Authentication configuration + auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com + # -- Datacatalog server config datacatalogServer: datacatalog: @@ -477,6 +512,8 @@ configmap: admin: endpoint: flyteadmin:81 insecure: true + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret # -- Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) # Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) @@ -567,6 +604,13 @@ configmap: show-source: true level: 4 + remoteData: + remoteData: + region: "us-east-1" + scheme: "local" + signedUrls: + durationMinutes: 3 + # -- Resource manager configuration resource_manager: # -- resource manager configuration From f74ee3f75ab485ded9a9c824c65e46937d3ae848 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 11 May 2021 09:53:48 -0700 Subject: [PATCH 42/92] update flytepropeller Signed-off-by: Haytham Abuelfutuh --- helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/values.yaml b/helm/values.yaml index 74679b6d78..2932a417e6 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -105,7 +105,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: ghcr.io/flyteorg/flytepropeller - tag: v0.10.7 + tag: v0.10.9 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: From b6247b1e61993f78e071d43d524469333bce630d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Wed, 12 May 2021 00:18:23 +0200 Subject: [PATCH 43/92] Add Pod Webhook deployment to Helm chart (#972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Pod Webhook deployment to Helm chart Based on a8c01e5a Pod Webhook deployment & Doc (#857) Signed-off-by: Sören Brunk * Remove K8s Job for Pod Webhook Signed-off-by: Haytham Abuelfutuh Co-authored-by: Haytham Abuelfutuh Signed-off-by: Haytham Abuelfutuh --- helm/templates/_helpers.tpl | 4 + helm/templates/propeller/deployment.yaml | 9 ++ helm/templates/propeller/webhook.yaml | 158 +++++++++++++++++++++++ helm/values.yaml | 23 +++- 4 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 helm/templates/propeller/webhook.yaml diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 3ae15cccf4..153591e02f 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -60,6 +60,10 @@ helm.sh/chart: {{ include "flyte.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} +{{- define "flyte-pod-webhook.name" -}} +flyte-pod-webhook +{{- end -}} + {{- define "flyteconsole.name" -}} flyteconsole diff --git a/helm/templates/propeller/deployment.yaml b/helm/templates/propeller/deployment.yaml index 7b5ae7e63a..ce9d6cc805 100644 --- a/helm/templates/propeller/deployment.yaml +++ b/helm/templates/propeller/deployment.yaml @@ -1,3 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: {{ template "flyte.namespace" . }} +type: Opaque +stringData: + client_secret: foobar +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/helm/templates/propeller/webhook.yaml b/helm/templates/propeller/webhook.yaml new file mode 100644 index 0000000000..b0e018b527 --- /dev/null +++ b/helm/templates/propeller/webhook.yaml @@ -0,0 +1,158 @@ +{{- if .Values.webhook.enabled }} +# Create an empty secret that the first propeller pod will populate +apiVersion: v1 +kind: Secret +metadata: + name: flyte-pod-webhook + namespace: {{ template "flyte.namespace" . }} +type: Opaque +--- +# Create the actual deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "flyte-pod-webhook.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: + app: {{ template "flyte-pod-webhook.name" . }} +spec: + selector: + matchLabels: + app: {{ template "flyte-pod-webhook.name" . }} + template: + metadata: + labels: + app: {{ template "flyte-pod-webhook.name" . }} + app.kubernetes.io/name: {{ template "flyte-pod-webhook.name" . }} + app.kubernetes.io/version: {{ .Values.flytepropeller.image.tag }} + annotations: + configChecksum: {{ include (print .Template.BasePath "/propeller/configmap.yaml") . | sha256sum | trunc 63 | quote }} + {{- with .Values.flyteadmin.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "flyte-pod-webhook.name" . }} +{{- if .Values.webhook.enabled }} + initContainers: + - name: generate-secrets + image: "{{ .Values.flytepropeller.image.repository }}:{{ .Values.flytepropeller.image.tag }}" + imagePullPolicy: "{{ .Values.flytepropeller.image.pullPolicy }}" + command: + - flytepropeller + args: + - webhook + - init-certs + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config +{{- end }} + containers: + - name: webhook + image: "{{ .Values.flytepropeller.image.repository }}:{{ .Values.flytepropeller.image.tag }}" + imagePullPolicy: "{{ .Values.flytepropeller.image.pullPolicy }}" + command: + - flytepropeller + args: + - webhook + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + readOnly: true + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: config-volume + configMap: + name: flyte-propeller-config + - name: webhook-certs + secret: + secretName: flyte-pod-webhook +--- +# Service +apiVersion: v1 +kind: Service +metadata: + name: {{ template "flyte-pod-webhook.name" . }} + namespace: {{ template "flyte.namespace" . }} + {{- with .Values.webhook.service.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + app: {{ template "flyte-pod-webhook.name" . }} + ports: + - name: https + protocol: TCP + port: 443 + targetPort: 9443 +--- +# Create a ClusterRole for the webhook +# https://kubernetes.io/docs/admin/authorization/rbac/ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "flyte-pod-webhook.name" . }} + namespace: {{ template "flyte.namespace" . }} +rules: + - apiGroups: + - "*" + resources: + - mutatingwebhookconfigurations + - secrets + - pods + verbs: + - get + - create + - update + - patch +--- +{{- if .Values.webhook.serviceAccount.create }} +# Create a Service Account for webhook +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "flyte-pod-webhook.name" . }} + namespace: {{ template "flyte.namespace" . }} + {{- with .Values.webhook.serviceAccount.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end}} +{{- end }} +--- +# Create a binding from Role -> ServiceAccount +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "flyte-pod-webhook.name" . }} + namespace: {{ template "flyte.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "flyte-pod-webhook.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "flyte-pod-webhook.name" . }} + namespace: {{ template "flyte.namespace" . }} +{{- end }} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 2932a417e6..1bcb41a2e4 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -105,7 +105,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: ghcr.io/flyteorg/flytepropeller - tag: v0.10.9 + tag: v0.10.10 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -170,6 +170,24 @@ flyteconsole: # -- affinity for Flyteconsole deployment affinity: {} +# +# WEBHOOK SETTINGS +# + +webhook: + # -- enable or disable secrets webhook + enabled: true + # -- Configuration for service accounts for the webhook + serviceAccount: + # -- Should a service account be created for the webhook + create: true + # -- Annotations for ServiceAccount attached to the webhook + annotations: {} + # -- Service settings for the webhook + service: + annotations: + projectcontour.io/upstream-protocol.h2c: grpc + type: ClusterIP # ---------------------------------------------- # Sandbox Configuration @@ -569,6 +587,9 @@ configmap: type: bucket rate: 10 capacity: 100 + webhook: + certDir: /etc/webhook/certs + serviceName: flyte-pod-webhook # -- For Workflow store use configuration [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/workflowstore#Config) # From 4bd12c3825401b2238eadb0adce3cd3674bfc28c Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Wed, 31 Mar 2021 11:59:55 -0400 Subject: [PATCH 44/92] dark theme updates (#883) Signed-off-by: cosmicBboy Signed-off-by: Haytham Abuelfutuh --- rsts/_static/custom.css | 80 +++++++++++++++++++++++++++++++++++++++++ rsts/conf.py | 2 +- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/rsts/_static/custom.css b/rsts/_static/custom.css index 13c39dba3f..d9851b7d8f 100644 --- a/rsts/_static/custom.css +++ b/rsts/_static/custom.css @@ -6,6 +6,86 @@ h1, h2, h3, h4, h5, h6 { max-width: 30%; } + .sidebar-tree .reference.external:after { content: none; } + +.sphx-glr-thumbcontainer { + background-color: transparent; + border: transparent; +} + +.sphx-glr-thumbcontainer:hover { + border: transparent; +} + +div.sphx-glr-download a { + color:white; + background-color: #9d68e4cf; + background-image: none; + border: 1px solid #9d68e4cf; +} + +div.sphx-glr-download a:hover { + background-color: #8b48e2cf; + box-shadow: none; +} + +div.sphx-glr-thumbcontainer a.headerlink { + display: none; +} + +div.sphx-glr-thumbcontainer:hover { + border-color: white; + box-shadow: none; +} + +.sphx-glr-script-out .highlight pre { + background-color: #f8f8f8; +} + +p.sphx-glr-script-out { + padding-top: 0em; +} + +.search__outer::-webkit-scrollbar-track { + border-radius: 0px; +} + +@media (prefers-color-scheme: dark) { + .search__outer { + background-color: #131416 !important; + border: 1px solid #131416 !important; + } + .search__outer__input { + background-color: #1a1c1e !important; + } + .search__result__single { + border-bottom: #303335 !important; + } + .outer_div_page_results:hover { + background-color: black; + } + .search__result__title, .rtd_ui_search_subtitle { + color: #9D68E4 !important; + border-bottom: 1px solid #9D68E4 !important; + } + .search__outer .search__result__title span, .search__outer .search__result__content span { + background-color: #9d68e454; + } + .search__result__subheading, .search__result__content { + color: #ffffffd9 !important; + } + .search__outer::-webkit-scrollbar-track { + background-color: #131416 !important; + } + .rtd__search__credits { + background-color: #1a1c1e !important; + border: 1px solid #1a1c1e !important; + color: #81868d !important; + } + .rtd__search__credits a, .search__error__box { + color: #9ca0a5 !important; + } + } diff --git a/rsts/conf.py b/rsts/conf.py index 9bfce8e5e9..f778cbe4de 100644 --- a/rsts/conf.py +++ b/rsts/conf.py @@ -109,7 +109,7 @@ templates_path = ["_templates"] pygments_style = "tango" -pygments_dark_style = "paraiso-dark" +pygments_dark_style = "native" html_css_files = [ "custom.css", From 6b8ed62d7d6c7a68acbc8b2a79914d26701104ae Mon Sep 17 00:00:00 2001 From: Yuvraj <10830562+evalsocket@users.noreply.github.com> Date: Thu, 1 Apr 2021 23:57:51 +0530 Subject: [PATCH 45/92] Release automation addedd for flyte definition (#835) * wip: added script for release automation Signed-off-by: yuvraj * wip: added manual github workflow Signed-off-by: Yuvraj * wip: added release workflow Signed-off-by: Yuvraj * wip: fix e2e testcase Signed-off-by: Yuvraj * wip: small fixes Signed-off-by: Yuvraj * wip: revert version in kustomize Signed-off-by: Yuvraj * wip: revert kustomize changes Signed-off-by: Yuvraj * wip: added generated kustomize output Signed-off-by: yuvraj * update latest version of component Signed-off-by: Yuvraj * wip: added generated kustomize output Signed-off-by: yuvraj * wip: added message input in workflow Signed-off-by: yuvraj * wip: remove release automation from kustomize command Signed-off-by: yuvraj * wip: generate kustomize output Signed-off-by: yuvraj * wip: revert webhook and build kustomize output Signed-off-by: yuvraj * updated release workflow Signed-off-by: yuvraj * fix typo Signed-off-by: yuvraj Signed-off-by: Haytham Abuelfutuh --- .github/workflows/release-kustomize.yml | 42 +++++++++++++++++++ Makefile | 6 ++- deployment/eks/flyte_generated.yaml | 22 +++++----- deployment/gcp/flyte_generated.yaml | 22 +++++----- deployment/sandbox/flyte_generated.yaml | 22 +++++----- deployment/test/flyte_generated.yaml | 18 ++++---- kustomize/overlays/eks/kustomization.yaml | 12 +++--- kustomize/overlays/gcp/kustomization.yaml | 12 +++--- kustomize/overlays/sandbox/kustomization.yaml | 10 ++--- kustomize/overlays/test/kustomization.yaml | 10 ++--- script/release.sh | 13 ++++++ 11 files changed, 124 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/release-kustomize.yml create mode 100755 script/release.sh diff --git a/.github/workflows/release-kustomize.yml b/.github/workflows/release-kustomize.yml new file mode 100644 index 0000000000..ca0f014afa --- /dev/null +++ b/.github/workflows/release-kustomize.yml @@ -0,0 +1,42 @@ +name: Publish flyte defination file + +on: + workflow_dispatch: + +jobs: + update-flyte-releases: + name: Update Flyte components + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Build kustomize + run: | + make release_automation + make kustomize + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.FLYTE_BOT_PAT }} + commit-message: Update Flyte Components + committer: Flyte-Bot + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: true + branch: flyte-bot-update-releases + delete-branch: true + title: 'Update Flyte components' + body: | + Updated flyte deployment + - Updated GCP Flyte deployment + - Updated EKS Flyte deployment + - Updated Sandbox Flyte deployment + - Updated TEST Flyte deployment + - Auto-generated by [flyte-bot] + labels: | + kustomize + team-reviewers: | + owners + maintainers + draft: false \ No newline at end of file diff --git a/Makefile b/Makefile index 3474016751..19cb24ee95 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,13 @@ pip-compile $(1) --upgrade --verbose endef .PHONY: kustomize -kustomize: +kustomize: KUSTOMIZE_VERSION=3.9.2 bash script/generate_kustomize.sh +.PHONY: release_automation +release_automation: + bash script/release.sh + .PHONY: deploy_sandbox deploy_sandbox: bash script/deploy.sh diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index efe1d383c1..31c70bd81b 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -8587,7 +8587,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/flyteorg/datacatalog:v0.3.0 + image: ghcr.io/flyteorg/datacatalog:v0.3.1 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -8610,7 +8610,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/datacatalog:v0.3.0 + image: ghcr.io/flyteorg/datacatalog:v0.3.1 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8667,7 +8667,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -8715,7 +8715,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8760,7 +8760,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8777,7 +8777,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8791,7 +8791,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8838,7 +8838,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: ghcr.io/flyteorg/flyteconsole:v0.19.6 + image: ghcr.io/flyteorg/flyteconsole:v0.19.7 name: flyteconsole ports: - containerPort: 8080 @@ -8882,7 +8882,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -9069,7 +9069,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -9119,7 +9119,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml index 79f76f71a4..0c3d03f416 100644 --- a/deployment/gcp/flyte_generated.yaml +++ b/deployment/gcp/flyte_generated.yaml @@ -8619,7 +8619,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/flyteorg/datacatalog:v0.3.0 + image: ghcr.io/flyteorg/datacatalog:v0.3.1 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -8642,7 +8642,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/datacatalog:v0.3.0 + image: ghcr.io/flyteorg/datacatalog:v0.3.1 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8699,7 +8699,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -8747,7 +8747,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8792,7 +8792,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8809,7 +8809,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8823,7 +8823,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8870,7 +8870,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: ghcr.io/flyteorg/flyteconsole:v0.19.6 + image: ghcr.io/flyteorg/flyteconsole:v0.19.7 name: flyteconsole ports: - containerPort: 8080 @@ -8914,7 +8914,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -9101,7 +9101,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -9151,7 +9151,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 5f162c1391..58f495184c 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -2722,7 +2722,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/flyteorg/datacatalog:v0.3.0 + image: ghcr.io/flyteorg/datacatalog:v0.3.1 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -2740,7 +2740,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/datacatalog:v0.3.0 + image: ghcr.io/flyteorg/datacatalog:v0.3.1 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -2797,7 +2797,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -2851,7 +2851,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -2902,7 +2902,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -2918,7 +2918,7 @@ spec: - seed-projects - flytesnacks - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -2932,7 +2932,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -2979,7 +2979,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: ghcr.io/flyteorg/flyteconsole:v0.19.6 + image: ghcr.io/flyteorg/flyteconsole:v0.19.7 name: flyteconsole ports: - containerPort: 8080 @@ -3023,7 +3023,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -3328,7 +3328,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -3512,7 +3512,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 2296b79bbd..2a10bff71e 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -582,7 +582,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/flyteorg/datacatalog:v0.3.0 + image: ghcr.io/flyteorg/datacatalog:v0.3.1 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -600,7 +600,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/datacatalog:v0.3.0 + image: ghcr.io/flyteorg/datacatalog:v0.3.1 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -657,7 +657,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -705,7 +705,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -756,7 +756,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -772,7 +772,7 @@ spec: - seed-projects - flytetester - flytesnacks - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -786,7 +786,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.0 + image: ghcr.io/flyteorg/flyteadmin:v0.4.2 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -843,7 +843,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -948,7 +948,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.8 + image: ghcr.io/flyteorg/flytepropeller:v0.7.10 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/kustomize/overlays/eks/kustomization.yaml b/kustomize/overlays/eks/kustomization.yaml index 21b3f2ef07..a8322c1299 100644 --- a/kustomize/overlays/eks/kustomization.yaml +++ b/kustomize/overlays/eks/kustomization.yaml @@ -21,21 +21,21 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.0 # override the tag + newTag: v0.4.2 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.19.6 # override the tag + newTag: v0.19.7 # FLYTECONSOLE_TAG the tag newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.3.0 # override the tag + newTag: v0.3.1 # DATACATALOG_TAG override the tag newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.7.8 # override the tag + newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.7.8 # override the tag - newName: ghcr.io/flyteorg/flytepropeller # override the name + newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newName: ghcr.io/flyteorg/flytepropeller # override the name \ No newline at end of file diff --git a/kustomize/overlays/gcp/kustomization.yaml b/kustomize/overlays/gcp/kustomization.yaml index a209c3bf86..802209f3b8 100644 --- a/kustomize/overlays/gcp/kustomization.yaml +++ b/kustomize/overlays/gcp/kustomization.yaml @@ -23,21 +23,21 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.0 # override the tag + newTag: v0.4.2 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.19.6 # override the tag + newTag: v0.19.7 # FLYTECONSOLE_TAG the tag newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.3.0 # override the tag + newTag: v0.3.1 # DATACATALOG_TAG override the tag newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.7.8 # override the tag + newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.7.8 # override the tag - newName: ghcr.io/flyteorg/flytepropeller # override the name + newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newName: ghcr.io/flyteorg/flytepropeller # override the name \ No newline at end of file diff --git a/kustomize/overlays/sandbox/kustomization.yaml b/kustomize/overlays/sandbox/kustomization.yaml index 50d2f74c61..e0a74e5d53 100644 --- a/kustomize/overlays/sandbox/kustomization.yaml +++ b/kustomize/overlays/sandbox/kustomization.yaml @@ -25,23 +25,23 @@ resources: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.0 # override the tag + newTag: v0.4.2 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.19.6 # override the tag + newTag: v0.19.7 # FLYTECONSOLE_TAG the tag newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.3.0 # override the tag + newTag: v0.3.1 # DATACATALOG_TAG override the tag newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.7.8 # override the tag + newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.7.8 # override the tag + newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Override postgres image to use alpine based (rather smaller) docker image - name: postgres diff --git a/kustomize/overlays/test/kustomization.yaml b/kustomize/overlays/test/kustomization.yaml index 6bb67c1098..943319bab1 100644 --- a/kustomize/overlays/test/kustomization.yaml +++ b/kustomize/overlays/test/kustomization.yaml @@ -19,23 +19,23 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.0 # override the tag + newTag: v0.4.2 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.19.6 # override the tag + newTag: v0.19.7 # FLYTECONSOLE_TAG the tag newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.3.0 # override the tag + newTag: v0.3.1 # DATACATALOG_TAG override the tag newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.7.8 # override the tag + newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.7.8 # override the tag + newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Override postgres image to use alpine based (rather smaller) docker image - name: postgres diff --git a/script/release.sh b/script/release.sh new file mode 100755 index 0000000000..b605cd9b0d --- /dev/null +++ b/script/release.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex + +FLYTEADMIN_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteadmin/releases/latest" | jq -r .tag_name) +DATACATALOG_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/datacatalog/releases/latest" | jq -r .tag_name) +FLYTECONSOLE_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteconsole/releases/latest" | jq -r .tag_name) +FLYTEPROPELLER_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytepropeller/releases/latest" | jq -r .tag_name) + +grep -rlZ "newTag:[^P]*# FLYTEADMIN_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTEADMIN_TAG/newTag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG/g" +grep -rlZ "newTag:[^P]*# DATACATALOG_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# DATACATALOG_TAG/newTag: ${DATACATALOG_TAG} # DATACATALOG_TAG/g" +grep -rlZ "newTag:[^P]*# FLYTECONSOLE_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTECONSOLE_TAG/newTag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG/g" +grep -rlZ "newTag:[^P]*# FLYTEPROPELLER_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTEPROPELLER_TAG/newTag: ${FLYTEPROPELLER_TAG} # FLYTEPROPELLER_TAG/g" From 0781077bfbc1d8dcb88b4f8884f0657139a7337a Mon Sep 17 00:00:00 2001 From: Flyte Bot Date: Thu, 1 Apr 2021 14:41:11 -0700 Subject: [PATCH 46/92] Milestone 0.12.0 (#888) * Update Flyte Components Signed-off-by: Flyte-Bot Signed-off-by: Haytham Abuelfutuh * update endtoend to flytekit 0.17 Signed-off-by: wild-endeavor Signed-off-by: Haytham Abuelfutuh * empty changelog Signed-off-by: wild-endeavor Signed-off-by: Haytham Abuelfutuh * Update ChangeLog Signed-off-by: Haytham Abuelfutuh * before pull from flyte-bot-update-releases Signed-off-by: Haytham Abuelfutuh Co-authored-by: EngHabu Co-authored-by: wild-endeavor Co-authored-by: HackMD Signed-off-by: Haytham Abuelfutuh --- CHANGELOG/CHANGELOG-v0.12.0.md | 35 +++++++++++++++++++ deployment/eks/flyte_generated.yaml | 16 ++++----- deployment/gcp/flyte_generated.yaml | 16 ++++----- deployment/sandbox/flyte_generated.yaml | 16 ++++----- deployment/test/flyte_generated.yaml | 14 ++++---- end2end/tests/endtoend.yaml | 2 +- kustomize/overlays/eks/kustomization.yaml | 6 ++-- kustomize/overlays/gcp/kustomization.yaml | 6 ++-- kustomize/overlays/sandbox/kustomization.yaml | 6 ++-- kustomize/overlays/test/kustomization.yaml | 6 ++-- 10 files changed, 79 insertions(+), 44 deletions(-) create mode 100644 CHANGELOG/CHANGELOG-v0.12.0.md diff --git a/CHANGELOG/CHANGELOG-v0.12.0.md b/CHANGELOG/CHANGELOG-v0.12.0.md new file mode 100644 index 0000000000..742cb4a12f --- /dev/null +++ b/CHANGELOG/CHANGELOG-v0.12.0.md @@ -0,0 +1,35 @@ +# General +* Added CoPilot configuration to sandbox manifest +* Updated and streamlined documentation navigation and themes. Better content organization coming soon! + +# Performance +* Improved execution performance including: + - Reduced cache lookups + - Improved GetWorkflowExecution performance + - Capped max number of nodes in each propeller round + - Misc. propeller performance tweaks +* TaskTemplate offloading + +# Housekeeping +* Migrated Datacatalog protobuf definitions to flyteidl [thanks @tnsetting] +* Upgraded stow version used in flytestdlib +* Moved off lyft kubernetes forks and onto official kubernetes library dependencies +* Revamped pod tasks to use official kubernetes python client library for defining PodSpecs + +# Events +* Richer event metadata for task executions +* Better merging of custom info across task events + +# Bug fixes +* Resolved non-backwards protobuf role changes that prevented launching single task executions [thanks @kanterov] +* Better handling of large workflows and errors in flytepropeller + +# Flytekit (Python) +* Access to secrets +* Bug fixes around the 0.16 release. + * Use original FlyteFile/FlyteDirectory + * Fix serialization of pod specs in pod plugin [thanks @jeevb] + * Accept auth role arg in single task execution + * Fixed task resolver in map task + * Requests and limits added to ContainerTask [thanks @migueltol22] + diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index 31c70bd81b..de4d19286b 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -8667,7 +8667,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -8715,7 +8715,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8760,7 +8760,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8777,7 +8777,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8791,7 +8791,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8882,7 +8882,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -9069,7 +9069,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -9119,7 +9119,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml index 0c3d03f416..4e7ce1c0c5 100644 --- a/deployment/gcp/flyte_generated.yaml +++ b/deployment/gcp/flyte_generated.yaml @@ -8699,7 +8699,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -8747,7 +8747,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8792,7 +8792,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8809,7 +8809,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8823,7 +8823,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8914,7 +8914,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -9101,7 +9101,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -9151,7 +9151,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 58f495184c..8073b2a3f5 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -2797,7 +2797,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -2851,7 +2851,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -2902,7 +2902,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -2918,7 +2918,7 @@ spec: - seed-projects - flytesnacks - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -2932,7 +2932,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -3023,7 +3023,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -3328,7 +3328,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -3512,7 +3512,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 2a10bff71e..d0f417a8fa 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -657,7 +657,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -705,7 +705,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -756,7 +756,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -772,7 +772,7 @@ spec: - seed-projects - flytetester - flytesnacks - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -786,7 +786,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.2 + image: ghcr.io/flyteorg/flyteadmin:v0.4.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -843,7 +843,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -948,7 +948,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.7.10 + image: ghcr.io/flyteorg/flytepropeller:v0.10.0 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/end2end/tests/endtoend.yaml b/end2end/tests/endtoend.yaml index cb9d16e8fe..7cd2cb6a50 100644 --- a/end2end/tests/endtoend.yaml +++ b/end2end/tests/endtoend.yaml @@ -11,7 +11,7 @@ spec: command: - bash - -c - image: ghcr.io/flyteorg/flytetools:78c917fb8d5dab74f5ff5f02d92e43d53ed4e7c3 + image: ghcr.io/flyteorg/flytetools:2638d4444add18096a688bc458d963cd3997e52e imagePullPolicy: IfNotPresent name: flytetester resources: diff --git a/kustomize/overlays/eks/kustomization.yaml b/kustomize/overlays/eks/kustomization.yaml index a8322c1299..4457715cff 100644 --- a/kustomize/overlays/eks/kustomization.yaml +++ b/kustomize/overlays/eks/kustomization.yaml @@ -21,7 +21,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.2 # FLYTEADMIN_TAG override the tag + newTag: v0.4.7 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -33,9 +33,9 @@ images: newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name \ No newline at end of file diff --git a/kustomize/overlays/gcp/kustomization.yaml b/kustomize/overlays/gcp/kustomization.yaml index 802209f3b8..7fcca6183f 100644 --- a/kustomize/overlays/gcp/kustomization.yaml +++ b/kustomize/overlays/gcp/kustomization.yaml @@ -23,7 +23,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.2 # FLYTEADMIN_TAG override the tag + newTag: v0.4.7 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -35,9 +35,9 @@ images: newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name \ No newline at end of file diff --git a/kustomize/overlays/sandbox/kustomization.yaml b/kustomize/overlays/sandbox/kustomization.yaml index e0a74e5d53..bb1e00f598 100644 --- a/kustomize/overlays/sandbox/kustomization.yaml +++ b/kustomize/overlays/sandbox/kustomization.yaml @@ -25,7 +25,7 @@ resources: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.2 # FLYTEADMIN_TAG override the tag + newTag: v0.4.7 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -37,11 +37,11 @@ images: newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Override postgres image to use alpine based (rather smaller) docker image - name: postgres diff --git a/kustomize/overlays/test/kustomization.yaml b/kustomize/overlays/test/kustomization.yaml index 943319bab1..1b4e568464 100644 --- a/kustomize/overlays/test/kustomization.yaml +++ b/kustomize/overlays/test/kustomization.yaml @@ -19,7 +19,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.2 # FLYTEADMIN_TAG override the tag + newTag: v0.4.7 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -31,11 +31,11 @@ images: newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Override postgres image to use alpine based (rather smaller) docker image - name: postgres From d84ade5d6c0412783f42ee72282929ea562692d0 Mon Sep 17 00:00:00 2001 From: SandraGH5 <80421934+SandraGH5@users.noreply.github.com> Date: Fri, 2 Apr 2021 14:21:03 -0700 Subject: [PATCH 47/92] Update index.rst (#884) * Update index.rst Add Flyte users to Who's Using Flyte and Meet Flyte. * Update rsts/index.rst Co-authored-by: Ketan Umare <16888709+kumare3@users.noreply.github.com> * Update index.rst Co-authored-by: Ketan Umare <16888709+kumare3@users.noreply.github.com> Signed-off-by: Haytham Abuelfutuh --- rsts/index.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rsts/index.rst b/rsts/index.rst index 4bd7d9cafd..4fee64cdd9 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -43,7 +43,7 @@ Meet Flyte Flyte is an open-source, container-native, structured programming and distributed processing platform. It enables highly concurrent, scalable and maintainable workflows for machine learning and data processing. -Created at `Lyft `__, Flyte provides first class support for Python, Java, and Scala, and is built directly on Kubernetes for all the benefits containerization provides: portability, scalability, and reliability. +Created at `Lyft `__ in collaboration with Spotify, Freenome and many others, Flyte provides first class support for Python, Java, and Scala, and is built directly on Kubernetes for all the benefits containerization provides: portability, scalability, and reliability. The core unit of execution in Flyte is the ``task``, which you can easily write with the Flytekit Python SDK: @@ -94,6 +94,8 @@ Who's Using Flyte? At `Lyft `__, Flyte has served production model training and data processing for over four years, becoming the de-facto platform for the Pricing, Locations, ETA, Mapping teams, Growth, Autonomous and other teams. +For the most current list of Flyte's deployments, please click `here `_. + Next Steps ---------- From aca3089d66d79589a3dfdc6bb0a3ad197fb8d49d Mon Sep 17 00:00:00 2001 From: Ketan Umare <16888709+kumare3@users.noreply.github.com> Date: Fri, 2 Apr 2021 21:20:12 -0700 Subject: [PATCH 48/92] Dashboard updates (#886) * Adding additional metrics in Flyteadmin dashboard Signed-off-by: Ketan Umare * Flytepropeller metrics added Signed-off-by: Ketan Umare * updated Signed-off-by: Ketan Umare * updated Signed-off-by: Ketan Umare * Added queue metrics Signed-off-by: Ketan Umare * updated name Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- .../prometheus/flyteadmin-dashboard.json | 10494 +++++++++++++++- .../prometheus/flytepropeller-dashboard.json | 2246 +++- stats/flyteadmin_dashboard.py | 96 +- stats/flytepropeller_dashboard.py | 333 +- 4 files changed, 13051 insertions(+), 118 deletions(-) diff --git a/deployment/stats/prometheus/flyteadmin-dashboard.json b/deployment/stats/prometheus/flyteadmin-dashboard.json index 507444c6aa..a120ec974c 100644 --- a/deployment/stats/prometheus/flyteadmin-dashboard.json +++ b/deployment/stats/prometheus/flyteadmin-dashboard.json @@ -77,6 +77,10302 @@ "renderer": "flot", "repeat": null, "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 2, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 3, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 4, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 5, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 6, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 7, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 8, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 9, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 10, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 11, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 12, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 13, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 14, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 15, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 16, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 17, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 18, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 19, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 20, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 21, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 22, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 23, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 24, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB task_executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 25, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 26, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 27, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 28, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 29, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 30, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 31, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 32, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB task_executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 33, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 34, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 35, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 36, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 37, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 38, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 39, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 40, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB node_executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 41, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 42, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 43, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 44, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 45, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 46, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 47, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 48, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB node_executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 49, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 50, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 51, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 52, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 53, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 54, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 55, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 56, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB workflows ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 57, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 58, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 59, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 60, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 61, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 62, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 63, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 64, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB workflows ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 65, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 66, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 67, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 68, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 69, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 70, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 71, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 72, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB launch_plans ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 73, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 74, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 75, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 76, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 77, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 78, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 79, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 80, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB launch_plans ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 81, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 82, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 83, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 84, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 85, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 86, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 87, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 88, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB project ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 89, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 90, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 91, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 92, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 93, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 94, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 95, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 96, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB project ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 97, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], "span": 4, "stack": false, "steppedLine": false, @@ -190,7 +10486,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 2, + "id": 98, "interval": null, "isNew": true, "legend": { @@ -310,7 +10606,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 3, + "id": 99, "interval": null, "isNew": true, "legend": { @@ -351,7 +10647,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:create_execution:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:create_execution:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -381,7 +10677,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -426,7 +10722,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 4, + "id": 100, "interval": null, "isNew": true, "legend": { @@ -574,7 +10870,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 5, + "id": 101, "interval": null, "isNew": true, "legend": { @@ -694,7 +10990,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 6, + "id": 102, "interval": null, "isNew": true, "legend": { @@ -735,7 +11031,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:create_launch_plan:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:create_launch_plan:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -765,7 +11061,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -810,7 +11106,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 7, + "id": 103, "interval": null, "isNew": true, "legend": { @@ -958,7 +11254,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 8, + "id": 104, "interval": null, "isNew": true, "legend": { @@ -1078,7 +11374,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 9, + "id": 105, "interval": null, "isNew": true, "legend": { @@ -1119,7 +11415,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:create_task:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:create_task:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -1149,7 +11445,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1194,7 +11490,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 10, + "id": 106, "interval": null, "isNew": true, "legend": { @@ -1342,7 +11638,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 11, + "id": 107, "interval": null, "isNew": true, "legend": { @@ -1462,7 +11758,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 12, + "id": 108, "interval": null, "isNew": true, "legend": { @@ -1503,7 +11799,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:create_workflow:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:create_workflow:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -1533,7 +11829,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1578,7 +11874,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 13, + "id": 109, "interval": null, "isNew": true, "legend": { @@ -1726,7 +12022,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 14, + "id": 110, "interval": null, "isNew": true, "legend": { @@ -1846,7 +12142,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 15, + "id": 111, "interval": null, "isNew": true, "legend": { @@ -1887,7 +12183,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:create_node_execution_event:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:create_node_execution_event:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -1917,7 +12213,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1962,7 +12258,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 16, + "id": 112, "interval": null, "isNew": true, "legend": { @@ -2110,7 +12406,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 17, + "id": 113, "interval": null, "isNew": true, "legend": { @@ -2230,7 +12526,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 18, + "id": 114, "interval": null, "isNew": true, "legend": { @@ -2271,7 +12567,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:create_task_execution_event:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:create_task_execution_event:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -2301,7 +12597,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -2346,7 +12642,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 19, + "id": 115, "interval": null, "isNew": true, "legend": { @@ -2494,7 +12790,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 20, + "id": 116, "interval": null, "isNew": true, "legend": { @@ -2614,7 +12910,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 21, + "id": 117, "interval": null, "isNew": true, "legend": { @@ -2655,7 +12951,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:get_execution:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:get_execution:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -2685,7 +12981,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -2730,7 +13026,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 22, + "id": 118, "interval": null, "isNew": true, "legend": { @@ -2878,7 +13174,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 23, + "id": 119, "interval": null, "isNew": true, "legend": { @@ -2998,7 +13294,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 24, + "id": 120, "interval": null, "isNew": true, "legend": { @@ -3039,7 +13335,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:get_launch_plan:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:get_launch_plan:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -3069,7 +13365,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -3114,7 +13410,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 25, + "id": 121, "interval": null, "isNew": true, "legend": { @@ -3262,7 +13558,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 26, + "id": 122, "interval": null, "isNew": true, "legend": { @@ -3382,7 +13678,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 27, + "id": 123, "interval": null, "isNew": true, "legend": { @@ -3423,7 +13719,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:get_task:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:get_task:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -3453,7 +13749,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -3498,7 +13794,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 28, + "id": 124, "interval": null, "isNew": true, "legend": { @@ -3646,7 +13942,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 29, + "id": 125, "interval": null, "isNew": true, "legend": { @@ -3766,7 +14062,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 30, + "id": 126, "interval": null, "isNew": true, "legend": { @@ -3807,7 +14103,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:get_workflow:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:get_workflow:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -3837,7 +14133,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -3882,7 +14178,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 31, + "id": 127, "interval": null, "isNew": true, "legend": { @@ -4030,7 +14326,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 32, + "id": 128, "interval": null, "isNew": true, "legend": { @@ -4150,7 +14446,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 33, + "id": 129, "interval": null, "isNew": true, "legend": { @@ -4191,7 +14487,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:get_node_execution:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:get_node_execution:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -4221,7 +14517,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -4266,7 +14562,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 34, + "id": 130, "interval": null, "isNew": true, "legend": { @@ -4414,7 +14710,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 35, + "id": 131, "interval": null, "isNew": true, "legend": { @@ -4534,7 +14830,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 36, + "id": 132, "interval": null, "isNew": true, "legend": { @@ -4575,7 +14871,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:get_task_execution:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:get_task_execution:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -4605,7 +14901,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -4650,7 +14946,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 37, + "id": 133, "interval": null, "isNew": true, "legend": { @@ -4798,7 +15094,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 38, + "id": 134, "interval": null, "isNew": true, "legend": { @@ -4918,7 +15214,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 39, + "id": 135, "interval": null, "isNew": true, "legend": { @@ -4959,7 +15255,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:get_active_launch_plan:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:get_active_launch_plan:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -4989,7 +15285,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -5034,7 +15330,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 40, + "id": 136, "interval": null, "isNew": true, "legend": { @@ -5182,7 +15478,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 41, + "id": 137, "interval": null, "isNew": true, "legend": { @@ -5302,7 +15598,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 42, + "id": 138, "interval": null, "isNew": true, "legend": { @@ -5343,7 +15639,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:list_execution:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:list_execution:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -5373,7 +15669,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -5418,7 +15714,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 43, + "id": 139, "interval": null, "isNew": true, "legend": { @@ -5566,7 +15862,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 44, + "id": 140, "interval": null, "isNew": true, "legend": { @@ -5686,7 +15982,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 45, + "id": 141, "interval": null, "isNew": true, "legend": { @@ -5727,7 +16023,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:list_launch_plan:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:list_launch_plan:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -5757,7 +16053,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -5802,7 +16098,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 46, + "id": 142, "interval": null, "isNew": true, "legend": { @@ -5950,7 +16246,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 47, + "id": 143, "interval": null, "isNew": true, "legend": { @@ -6070,7 +16366,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 48, + "id": 144, "interval": null, "isNew": true, "legend": { @@ -6111,7 +16407,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:list_task:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:list_task:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -6141,7 +16437,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -6186,7 +16482,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 49, + "id": 145, "interval": null, "isNew": true, "legend": { @@ -6334,7 +16630,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 50, + "id": 146, "interval": null, "isNew": true, "legend": { @@ -6454,7 +16750,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 51, + "id": 147, "interval": null, "isNew": true, "legend": { @@ -6495,7 +16791,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:list_workflow:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:list_workflow:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -6525,7 +16821,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -6570,7 +16866,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 52, + "id": 148, "interval": null, "isNew": true, "legend": { @@ -6718,7 +17014,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 53, + "id": 149, "interval": null, "isNew": true, "legend": { @@ -6838,7 +17134,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 54, + "id": 150, "interval": null, "isNew": true, "legend": { @@ -6879,7 +17175,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:list_node_execution:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:list_node_execution:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -6909,7 +17205,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -6954,7 +17250,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 55, + "id": 151, "interval": null, "isNew": true, "legend": { @@ -7102,7 +17398,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 56, + "id": 152, "interval": null, "isNew": true, "legend": { @@ -7222,7 +17518,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 57, + "id": 153, "interval": null, "isNew": true, "legend": { @@ -7263,7 +17559,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:list_task_execution:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:list_task_execution:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -7293,7 +17589,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -7338,7 +17634,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 58, + "id": 154, "interval": null, "isNew": true, "legend": { @@ -7486,7 +17782,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 59, + "id": 155, "interval": null, "isNew": true, "legend": { @@ -7606,7 +17902,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 60, + "id": 156, "interval": null, "isNew": true, "legend": { @@ -7647,7 +17943,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:admin:list_active_launch_plan:duration_ms[5m])) by (quantile)", + "expr": "sum(flyte:admin:list_active_launch_plan:duration_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -7677,7 +17973,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, diff --git a/deployment/stats/prometheus/flytepropeller-dashboard.json b/deployment/stats/prometheus/flytepropeller-dashboard.json index 3dbd64bc4d..457765c18d 100644 --- a/deployment/stats/prometheus/flytepropeller-dashboard.json +++ b/deployment/stats/prometheus/flytepropeller-dashboard.json @@ -189,7 +189,7 @@ "targets": [ { "datasource": "", - "expr": "sum(rate(flyte:propeller:all:round:abort_error[5m]))*300", + "expr": "sum(rate(flyte:propeller:all:round:abort_error[5m]))", "format": "time_series", "hide": false, "instant": false, @@ -295,7 +295,7 @@ "targets": [ { "datasource": "", - "expr": "sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))*300", + "expr": "sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))", "format": "time_series", "hide": false, "instant": false, @@ -985,7 +985,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "percent", "label": null, "logBase": 1, "max": null, @@ -1781,7 +1781,7 @@ "renderer": "flot", "repeat": null, "seriesOverrides": [], - "span": 4, + "span": 3, "stack": false, "steppedLine": false, "targets": [ @@ -1887,7 +1887,7 @@ "renderer": "flot", "repeat": null, "seriesOverrides": [], - "span": 4, + "span": 3, "stack": false, "steppedLine": false, "targets": [ @@ -1993,7 +1993,7 @@ "renderer": "flot", "repeat": null, "seriesOverrides": [], - "span": 4, + "span": 3, "stack": false, "steppedLine": false, "targets": [ @@ -2060,11 +2060,2245 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 19, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "system error", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "user error", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "user error", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "node event recording count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] } ], "repeat": null, "showTitle": true, "title": "Node Metrics" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 20, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "wf event recording latency success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 21, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "wf event recording count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 22, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "node event recording latency success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 23, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "node event recording count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 24, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "task event recording latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 25, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success wf", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "task event recording count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 26, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dynamic workflow build latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 27, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dynamic workflow build count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 28, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "hit", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "miss", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Admin Launcher cache", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Perf metrics" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 29, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "wf update etcD latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 30, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "etcD writes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 31, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "etcD write conflicts", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 32, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "etcD write fail", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "etcD write metrics" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 33, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_adds[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "WF Adds to main queue", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 34, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_depth[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Unprocessed Queue depth", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 35, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_retries[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Item retries", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 36, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "flyte:propeller:all:main_unfinished_work_s", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Seconds of unfinished work in progress", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 37, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_work_duration_us_sum[5m]) / rate(flyte:propeller:all:main_work_duration_us_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Workqueue work average duration", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 38, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_queue_latency_us_sum[5m]) / rate(flyte:propeller:all:main_queue_latency_us_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Duration for which an item stays in queue - avg", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "FlytePropeller Queue metrics" } ], "schemaVersion": 12, diff --git a/stats/flyteadmin_dashboard.py b/stats/flyteadmin_dashboard.py index e87d1d0473..c8e8aa819d 100644 --- a/stats/flyteadmin_dashboard.py +++ b/stats/flyteadmin_dashboard.py @@ -2,7 +2,7 @@ from grafanalib.core import ( Alert, AlertCondition, Dashboard, Graph, GreaterThan, OP_AND, OPS_FORMAT, Row, RTYPE_SUM, SECONDS_FORMAT, - SHORT_FORMAT, single_y_axis, Target, TimeRange, YAxes, YAxis, DataSourceInput + SHORT_FORMAT, single_y_axis, Target, TimeRange, YAxes, YAxis, DataSourceInput, MILLISECONDS_FORMAT ) # ------------------------------ @@ -38,6 +38,26 @@ class FlyteAdmin(object): "list_active_launch_plan", ] + ENTITIES = [ + "executions", + "task_executions", + "node_executions", + "workflows", + "launch_plans", + "project", + ] + + DB_OPS = [ + "get", + "list", + "create", + "update", + "list", + "list_identifiers", + "delete", + "exists", + ] + @staticmethod def error_codes(api: str, interval: int = 1) -> Graph: return Graph( @@ -101,11 +121,11 @@ def api_latency(api: str, interval: int = 1) -> Graph: dataSource=DATASOURCE, targets=[ Target( - expr=f'sum(rate(flyte:admin:{api}:duration_ms[{interval}m])) by (quantile)', + expr=f'sum(flyte:admin:{api}:duration_ms) by (quantile)', refId='A', ), ], - yAxes=single_y_axis(format=SECONDS_FORMAT), + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), ) @staticmethod @@ -120,6 +140,67 @@ def create_api_row(api: str, collapse: bool, interval: int = 1) -> Row: ] ) + @staticmethod + def db_latency(entity: str, op: str, interval: int = 1) -> Graph: + return Graph( + title=f"{op} Latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:admin:database:postgres:repositories:{entity}:{op}_ms) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def create_entity_db_row_latency(entity: str, collapse: bool, interval: int = 1) -> Row: + r = Row( + title=f"DB {entity} ops stats", + collapse=collapse, + panels=[], + ) + for op in FlyteAdmin.DB_OPS: + r.panels.append(FlyteAdmin.db_latency(entity, op=op, interval=interval)) + return r + + @staticmethod + def db_count(entity: str, op: str, interval: int = 1) -> Graph: + return Graph( + title=f"{op} Count Ops", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:admin:database:postgres:repositories:{entity}:{op}_ms_count[{interval}m]))', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def create_entity_db_count(entity: str, collapse: bool, interval: int = 1) -> Row: + r = Row( + title=f"DB {entity} ops stats", + collapse=collapse, + panels=[], + ) + for op in FlyteAdmin.DB_OPS: + r.panels.append(FlyteAdmin.db_count(entity, op=op, interval=interval)) + return r + + @staticmethod + def create_all_entity_db_rows(collapse: bool, interval: int = 1) -> typing.List[Row]: + rows = [] + for entity in FlyteAdmin.ENTITIES: + rows.append(FlyteAdmin.create_entity_db_row_latency(entity=entity, collapse=collapse, interval=interval)) + rows.append(FlyteAdmin.create_entity_db_count(entity=entity, collapse=collapse, interval=interval)) + return rows + @staticmethod def create_all_apis(interval: int = 5) -> typing.List[Row]: rows = [] @@ -127,6 +208,13 @@ def create_all_apis(interval: int = 5) -> typing.List[Row]: rows.append(FlyteAdmin.create_api_row(api, collapse=True, interval=interval)) return rows + @staticmethod + def create_all_rows(interval: int = 5) -> typing.List[Row]: + rows = [] + rows.extend(FlyteAdmin.create_all_entity_db_rows(collapse=True, interval=interval)) + rows.extend(FlyteAdmin.create_all_apis(interval)) + return rows + dashboard = Dashboard( tags=["flyte", "prometheus", "flyteadmin", "flyte-controlplane"], @@ -141,7 +229,7 @@ def create_all_apis(interval: int = 5) -> typing.List[Row]: ], editable=False, title="Flyte Admin Dashboard (via Prometheus)", - rows=FlyteAdmin.create_all_apis(interval=5), + rows=FlyteAdmin.create_all_rows(), description="Flyte Admin/Control Plane Dashboard. This is great for monitoring FlyteAdmin and the Service API.", ).auto_panel_ids() diff --git a/stats/flytepropeller_dashboard.py b/stats/flytepropeller_dashboard.py index f3cd01618d..bc306fee94 100644 --- a/stats/flytepropeller_dashboard.py +++ b/stats/flytepropeller_dashboard.py @@ -1,8 +1,9 @@ import typing + from grafanalib.core import ( - Alert, AlertCondition, Dashboard, Graph, - GreaterThan, OP_AND, OPS_FORMAT, Row, RTYPE_SUM, SECONDS_FORMAT, - SHORT_FORMAT, single_y_axis, Target, TimeRange, YAxes, YAxis, MILLISECONDS_FORMAT, DataSourceInput + Dashboard, Graph, Gauge, Stat, + OPS_FORMAT, Row, SHORT_FORMAT, single_y_axis, Target, YAxes, YAxis, MILLISECONDS_FORMAT, DataSourceInput, + PERCENT_FORMAT, NO_FORMAT, SECONDS_FORMAT ) # ------------------------------ @@ -102,7 +103,7 @@ def system_errors() -> Graph: dataSource=DATASOURCE, targets=[ Target( - expr='sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))*300', + expr='sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))', refId='A', ), ], @@ -119,7 +120,7 @@ def abort_errors() -> Graph: dataSource=DATASOURCE, targets=[ Target( - expr='sum(rate(flyte:propeller:all:round:abort_error[5m]))*300', + expr='sum(rate(flyte:propeller:all:round:abort_error[5m]))', refId='A', ), ], @@ -269,10 +270,7 @@ def metastore_cache_hit_percentage(interval: int) -> Graph: refId='A', ), ], - yAxes=YAxes( - YAxis(format=OPS_FORMAT), - YAxis(format=SHORT_FORMAT), - ), + yAxes=single_y_axis(format=PERCENT_FORMAT), ) @staticmethod @@ -346,6 +344,219 @@ def metastore_latencies(collapse: bool) -> Row: ] ) + @staticmethod + def admin_launcher_cache() -> Graph: + return Graph( + title="Admin Launcher cache", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))', + legendFormat="hit", + refId='A', + ), + + Target( + expr=f'sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))', + legendFormat="miss", + refId='B', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def dynamic_wf_build() -> typing.List[Graph]: + return [ + Graph( + title="Dynamic workflow build latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="Dynamic workflow build count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ] + + @staticmethod + def task_event_recording() -> typing.List[Graph]: + return [ + Graph( + title="task event recording latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="task event recording count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)', + legendFormat="success wf", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)', + legendFormat="failure", + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ] + + @staticmethod + def node_event_recording() -> typing.List[Graph]: + return [ + Graph( + title="node event recording latency success", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="node event recording count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)', + legendFormat="success", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)', + legendFormat="failure", + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ] + + @staticmethod + def wf_event_recording() -> typing.List[Graph]: + return [ + Graph( + title="wf event recording latency success", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="wf event recording count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)', + legendFormat="success", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)', + legendFormat="failure", + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ] + + @staticmethod + def wf_store_latency(collapse: bool) -> Row: + return Row( + title="etcD write metrics", + collapse=collapse, + panels=[ + Graph( + title="wf update etcD latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="etcD writes", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + Graph( + title="etcD write conflicts", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:wf_update_conflict[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + Graph( + title="etcD write fail", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:wf_update_failed[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ]) + + @staticmethod + def perf_metrics(collapse: bool) -> Row: + r = Row( + title="Perf metrics", + collapse=collapse, + panels=[], + ) + r.panels.extend(FlytePropeller.wf_event_recording()) + r.panels.extend(FlytePropeller.node_event_recording()) + r.panels.extend(FlytePropeller.task_event_recording()) + r.panels.extend(FlytePropeller.dynamic_wf_build()) + r.panels.append(FlytePropeller.admin_launcher_cache()) + return r + @staticmethod def metastore_metrics(interval: int, collapse: bool) -> Row: return Row( @@ -357,6 +568,106 @@ def metastore_metrics(interval: int, collapse: bool) -> Row: ], ) + @staticmethod + def node_errors() -> Graph: + return Graph( + title="node event recording count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))', + legendFormat="system error", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))', + legendFormat="user error", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))', + legendFormat="user error", + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ) + + @staticmethod + def queue_metrics(collapse: bool) -> Row: + return Row( + title="FlytePropeller Queue metrics", + collapse=collapse, + panels=[ + Graph( + title="WF Adds to main queue", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_adds[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + Graph( + title="Unprocessed Queue depth", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_depth[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + Graph( + title="Item retries", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_retries[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + Graph( + title="Seconds of unfinished work in progress", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'flyte:propeller:all:main_unfinished_work_s', + refId='A', + ), + ], + yAxes=single_y_axis(format=SECONDS_FORMAT), + ), + Graph( + title="Workqueue work average duration", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_work_duration_us_sum[5m]) / rate(flyte:propeller:all:main_work_duration_us_count[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SECONDS_FORMAT), + ), + Graph( + title="Duration for which an item stays in queue - avg", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_queue_latency_us_sum[5m]) / rate(flyte:propeller:all:main_queue_latency_us_count[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SECONDS_FORMAT), + ), + ], + ) + @staticmethod def node_metrics(collapse: bool) -> Row: return Row( @@ -366,6 +677,7 @@ def node_metrics(collapse: bool) -> Row: FlytePropeller.node_exec_latency(), FlytePropeller.node_input_latency(), FlytePropeller.node_event_recording_latency(), + FlytePropeller.node_errors(), ], ) @@ -393,6 +705,9 @@ def create_all_rows(interval: int = 5) -> typing.List[Row]: FlytePropeller.metastore_metrics(interval, True), FlytePropeller.metastore_latencies(True), FlytePropeller.node_metrics(True), + FlytePropeller.perf_metrics(True), + FlytePropeller.wf_store_latency(False), + FlytePropeller.queue_metrics(True), ] From bea33d0b5b3b3ea2b3e85b69f6311bd2747f7a79 Mon Sep 17 00:00:00 2001 From: Jeev B Date: Fri, 9 Apr 2021 11:42:07 -0700 Subject: [PATCH 49/92] Add init container that will pre-create buckets in minio (#905) * Add init container that will pre-create buckets in minio Signed-off-by: Jeev B * move minio tag override to overlay Signed-off-by: Jeev B Signed-off-by: Haytham Abuelfutuh --- deployment/sandbox/flyte_generated.yaml | 11 ++++++++++- deployment/test/flyte_generated.yaml | 11 ++++++++++- kustomize/base/addons/storage/storage.yaml | 11 ++++++++++- kustomize/overlays/sandbox/kustomization.yaml | 2 ++ kustomize/overlays/test/kustomization.yaml | 2 ++ 5 files changed, 34 insertions(+), 3 deletions(-) diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 8073b2a3f5..a6276ebb7b 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -3060,7 +3060,7 @@ spec: value: minio - name: MINIO_SECRET_KEY value: miniostorage - image: minio/minio:RELEASE.2020-12-16T05-05-17Z + image: minio/minio:RELEASE.2021-04-06T23-11-00Z name: minio ports: - containerPort: 9000 @@ -3068,6 +3068,15 @@ spec: volumeMounts: - mountPath: /data name: minio-storage + initContainers: + - command: + - mkdir + - /data/my-s3-bucket + image: minio/minio:RELEASE.2021-04-06T23-11-00Z + name: create-bucket + volumeMounts: + - mountPath: /data + name: minio-storage volumes: - emptyDir: {} name: minio-storage diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index d0f417a8fa..62205adc45 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -880,7 +880,7 @@ spec: value: minio - name: MINIO_SECRET_KEY value: miniostorage - image: minio/minio:RELEASE.2020-12-16T05-05-17Z + image: minio/minio:RELEASE.2021-04-06T23-11-00Z name: minio ports: - containerPort: 9000 @@ -888,6 +888,15 @@ spec: volumeMounts: - mountPath: /data name: minio-storage + initContainers: + - command: + - mkdir + - /data/my-s3-bucket + image: minio/minio:RELEASE.2021-04-06T23-11-00Z + name: create-bucket + volumeMounts: + - mountPath: /data + name: minio-storage volumes: - emptyDir: {} name: minio-storage diff --git a/kustomize/base/addons/storage/storage.yaml b/kustomize/base/addons/storage/storage.yaml index b14b37be81..132a338cb8 100644 --- a/kustomize/base/addons/storage/storage.yaml +++ b/kustomize/base/addons/storage/storage.yaml @@ -16,8 +16,17 @@ spec: volumes: - name: minio-storage emptyDir: {} + initContainers: + - image: minio/minio + name: create-bucket + command: + - mkdir + - /data/my-s3-bucket + volumeMounts: + - name: minio-storage + mountPath: /data containers: - - image: "minio/minio:RELEASE.2020-12-16T05-05-17Z" + - image: minio/minio name: minio env: - name: MINIO_ACCESS_KEY diff --git a/kustomize/overlays/sandbox/kustomization.yaml b/kustomize/overlays/sandbox/kustomization.yaml index bb1e00f598..76c85d0d30 100644 --- a/kustomize/overlays/sandbox/kustomization.yaml +++ b/kustomize/overlays/sandbox/kustomization.yaml @@ -46,3 +46,5 @@ images: # Override postgres image to use alpine based (rather smaller) docker image - name: postgres newTag: 10.16-alpine + - name: minio/minio + newTag: RELEASE.2021-04-06T23-11-00Z diff --git a/kustomize/overlays/test/kustomization.yaml b/kustomize/overlays/test/kustomization.yaml index 1b4e568464..ddf689f58c 100644 --- a/kustomize/overlays/test/kustomization.yaml +++ b/kustomize/overlays/test/kustomization.yaml @@ -40,3 +40,5 @@ images: # Override postgres image to use alpine based (rather smaller) docker image - name: postgres newTag: 10.16-alpine + - name: minio/minio + newTag: RELEASE.2021-04-06T23-11-00Z From 7604f90f71efdf4b9d1b7049bbe87ca1a05a2b59 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Sun, 11 Apr 2021 21:00:30 +0530 Subject: [PATCH 50/92] Add LF AI & Data Logo to README (#902) Signed-off-by: Haytham Abuelfutuh --- README.md | 2 +- rsts/images/flyte-and-lf.png | Bin 0 -> 166426 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 rsts/images/flyte-and-lf.png diff --git a/README.md b/README.md index 0bdf0ead81..90dcaac6fe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

- Flyte Logo + Flyte and LF AI & Data Logo

diff --git a/rsts/images/flyte-and-lf.png b/rsts/images/flyte-and-lf.png new file mode 100644 index 0000000000000000000000000000000000000000..47c8958805dcbe8a263900d79b02aef53b0d68f7 GIT binary patch literal 166426 zcmY&g2|SeB`|k=xA*4-G*-A-eD?%YcNM%dWLQ~et&M;Rg$(k%7Bq4jq$Tmh4lAY`f zW#7g=82q0zxxf7TeB8>docFxvd7kffcwN#|VPW3GykWxz7S#(%mp5!+nB1`8pN`Fp z@EQA7(#{PV_HR&CI(OCa&1j^99ENaLV$orrvUwh%@&zz_tc5pEg#R) z9k_Lzw1JWPiOjc~3cT$%Ejp6(ygT~3ejlt59uJ8az7U=^ZuO)XFFqYL;UAW9H^rvE zz*0Pu+R<#6=54akTvaSR&`@Pwk$%IyO?QqK-_Eejxt1!+!O5APsee?DQ+xg%&j0J} zp{^`U%|yqeN)h?ISxk-*u8hZ_!*<$NYE^i)A*K7&wEwK!8&h12ZLkd`rmyLKT%~zD zuaR~ksrJQiucATI5-X-@RQBT!ZTtO|<1!yp!g=)foLUO~B>&aBr1~8`OVoD9Px|WH zp6QP6u_=x(U9Be-1_-WU)xEt>=NOiDO2JU6<)6~NUA$QE;&~d!GG=Lh0@qZ+W@v?-au=F#{9UZFLN0gpKVy*H z){t{xqO`2KN<(!{gJtARtMAup4VLk64Hk^K^;5B&1W<9xL1+Gxt0nG85aeN8)-C3uWHUz+MuGIQ8)9`kVQrPU5D2C_})O%su8si zaRHt5Gm{S=#wSeh&CAaH=5vr;@$}?%obvPJ6p@;n=r*P}9ekJkSe`mqy=#OqgYAD0 z1iv`Tb?$!jwxG&R$JMZul$2!8nSE2qFI|;aCZ30`wBlN}l*qSb4W9izB)d~ra>tu9 zjrtEV({jnS>WV9>_7Bbrv`6byvzq8Ay;(NkvcQE@IUG-JbGYZ^JMlo*cNKk*Pr8tf z<{a@1Z4XwAd1b2ii5VMKAZqmc(u`g*a4xAVal!akmc&y2Hd>+4&nCU6`IbUS$g`e@kPkvM&^*{%f(Yygbj=yoo)~A zxry&fFgXA7ux@VeOk!qA!?F3c5Lxc{(YE7G`injlB?EHu&3RK%Js~~K7YHUy+olZUy+AhO@ zJhx%@;wvggsmpfFH|EFg87E(QA%sugv;LVkoH!sZ$35DT^Qzb@hks>I9yc4EGx4Ks z;j>ut>9j|&mWm6E?G@n~n*A-m@6Kr)yW9QycAt52ON@@`D0%piZeHHy!v>bq!D(re z!Rcv}q3NmG-V}Z_FsbrJ)Xx7SLC$Ju6O;M7DDu92jm zDLWmtuZzP(1HIIs)QpV!CP{c?;Rm`5nhDz}?+hu@^ll!dY|Pl0(jq02vGa?|?bUwb z37LFVi56Nm!sip+uLPK2o>tiFz8*DsTA`^M@cZWZ!^i9&^mwJS{gBVgH`5tubkkyS znG{{^9Em!|#qTF6%3Z?*z04=mTAB4f#Dv}?x6WN032D3)!nO%Vnexb=o}Lz6_VxDg zPG^7IxM|%}{QH`-z_iLTX5jsAtaU}HfELED=AvD{(;RQN!@ESMGVT73;oF;Z_d zq0_Rd>AdUKxM73zX7%4n<;^hVtbKXT%t^Kjwwud@Sadgjj_pypajWxb-eYH70^xCR z$}!Wv5#gZBY%a7&x%Ndf-2O;T!SU?P)Fx>L!o6}yHB{Z9g0?4pnXrXSk@x6~`6s6_ zo{f=6Urj=K@D>K!@okk`$Y343mnyxB(i9cZC; zo3SOkdAx`NrnbxiyLrNhY!|5bE63a8EFy-c2C5z`P2W8g{3cPtVx)18?y2MuuIjWq zPqRk!wJ&#ejonv~&z+aTCgr-qWT)Lcv2^kmdm{wM*(SD)o2j)G5j_?%4UtE2@rD%FJ!|z@>K9)oJvU^RgyY*Gl&agMN_Z>FMClMVo-z2)flIl$TeQ}? z=y29Y_fW1`^^9zTJTZG5aV+P2|Yy zy7!}RTp!&ZSDH8G)%?`4Yc6yApRo07`#j1dc;VQK>T`vgWIx#%%~5u=Hwy%}#{Ix2 zzR|Hvx+!6p95kG&qt?`~rkj0w|9CbtHnDWTj?BW0y}t=+VT6Mc$Vw28t5=HP?mz2HUx!tUc6pA;jJhn6(#4`8{`rZ8Jr?tnq(tvI2MtNZ94D}=S3_! zG?8_B%BYfsnmu)({T>T-v%Bca`T7HfuaqsI5Z+6XPfv9r zrsm?r%R$*$Q%YCo;~6 zG4Y7d8B3Z}st@ksqSSk6zAly0AaPrey!i=f9)3!sY4|g=rpQEPx`hcW%ent9*0d9& zV^O8g-5xd*C+bp=?3_DG^bm4eDxD225Xk%WY(1FXKYsjMo`9gB5!Yg~mzP)f_nPmy z-7n1}j0>}S6-rn|le;cMpHtr`HDQw!6i{F-G}$C*`)M<_B0xkm=8ir|okP0bkixNc z_!Ydmy%-1~9;{lVEdnb_?xk!|Rh+Wl`)h8bj>sk7;x3ahSZU<5C^=R3bd+g)VfVyD zt~?qawO#-)Qi)w6dza$>{S~h;VQ}eC54e;C7$V6 z$g&g3B8sG)S>{6J{&re_ZompZWW&~eyt;}tZsDiT@GC2)(fN>Qk1pPGVg5f@*S)~E zik}^`KCj9zu8s^$_;EX>O?ltQe#)wwk>PO~ue$)tQ>*%^rkf>kXk3(?V#X=`j@>Y- zAk?pAibFb+-Oy;GvATlF$=j9YBnJ}(70KI23f)ETlcwqAw7R%0Nh9z(jll=MUFRtH ztP{51a$DkS3cr7=DI{b^|NNNB$!+jZZ6qxxI_%UGFK%!(yHl`POHj4J~yFVe3J>~)v#`YP#XlM@MFpXE+uYO-=`OZ9F{>rbyLQ{EN6zJ~Wb zBe*?dvJC6?XG3^~{?Sga!P|`dIMX?nxRE(6d7Hrt(=R*iQk>OyCwE{!N3NYBA9z^` z7+-9;WEvgr;Q%LvC#?3R8a5W4_amx<7Z}`Lmv)m4T6Pc2rgC3t$Q!>R)Gpaa-!8Fi zF`cPx+Mwt6qv*h>NXRy<|N4{juk+(OZ)6o#o$Cf{a1S3@|4QhhCS^rE-gCG8`WLfD z-?_?kx%IlZcT&d0r1R(rgF6WO&@12-KScJvQ%l^bip_#X`OLzX8QT>g0;k8S$KeB! zm;2O6Szhbn!+qV08OKa#@5!`_r$t%r>waKg_Cr)|xuE8qu}5K_^UVz)>iqA=Tjt~@ zXYH;cDRN==C*8=U(kqu-zegDsn?0&D(9k?>)8D$?MUz3V%bcC^r|n4Ck&GgmM^fi^yG~oFA1mP^NnJoL{Yknq$urV7+9}vB`g2R$iDUS2PAS9H`(ufj zwyQ!LMJAS?&3COI!vAjGb%$yD#esshZqKO^IVPDqg#@(17yilm#X+JYFECM&Q;o~%T^!gWg7n?q z|J$hz8~!~K`QV|Jl1YX>A9_=<(OJJ)5y2ig;xPIqUKeA@X@C@-JcZU%SmzdyZ93srKVPlPxHX z4Vk^a%<%R;o~gV09%=E0$b7GFn2P+EbI46av@MF~sJ;+ByNUDXob& z7W)U*Y9AphV#an!8OnScF_!w?ly^5MqEG8gNv7O~L}T$OB55Qof4aS3UU=Qcy!Gak zyC!2&9$=jkUfB3(`0Ot?t#cAK19}<5B5|Yd3fOpF(Gxzoi|)=0W~OHMZ=r7XD)M<% zcS4w4!jFntabq-W2T=Nr#@+}Ngf|iOWK5C`N#4?wBzi(bD$m$0a}}4C@Ve}kU|yM) zyc2d7ov)qomc3!iuNJL8Zq0QnmwIe{!>4tq0>Rb^gYuQLyz2#@49=Q;lBADZo zi*McDEE&j>BcR`uFKu?`EnznlVd5EL zp;ib>Q?!2<-YXAjChoK&!>Bvn zug+L3$?T{kPLR8Pt=o`)E9t-K(=p`F!zkA~Jsrs=gAWek5ly=zk!$*ckXq0bjF|>< zsw2#g>uojLGoZXj15>l+2%t zdWMCmN8}!5h(t%;PL_KRlDgZ5+JjWr$JPtu#q(#@t>kSMXV(w6sTdVp-}mI}wI3Ub zucHH!5R^M*;ih85VDmdnh#bd4wRl2@yD`H^us|G1uk=uW-ZA$hE7w@UoVgCY)ZFNzIB~V5MAvousbu*S9bMN!olLn0 z8J$s%-2o1u)g+O^?tXbZ73&CcBOD0?wPFg(kAMwobvMLdR9|nu1Ddf&b59 z|8QtA6FRnJC#J}zy-r2Ld~Ajbw@4|J+Dcv5>&!kqGX$Ea$pi z+VJoow(;bVBO}ATedn+Eapi02S;-NvI|a+lOGb3(ranFoIvI^50yY+)>wbU^51eET zE=s|HcE`c;EXn#497 z)Hl(*6has=NtQRe?G&9IwDr@)g6R2_bqm>0x(f=5{lq{sTwCmM#?FK7^&{S-ZW`%S2NR4xIl`gLK{q~C{uS9TrR~ksMzmiB<>yOd4 zBVPnMT5`(H*e~9^^3bmOR{yaNSI<0^{2H8oxhn^HhY@KSkU4w%b4G$6yW#zf#!CS6 z3amxR$uq$3r`)t9+H(S4)z#Gz{e$K+c4bHux$_3hv+A&5hn|ZF#qjOP=wpIcw^|Pm zGi4-xJ6X3&-HN2HVsV;<`ftJfgTCdd`9g~Y7fVc^i{1CBd9&cJ6Q}g&gfK6Rmx?m{ zC!-Fm-^Ev(U*uRRDS=57&gr>F5PRd{Xk}8W=j-H;@4f~b7k^&hfsaRAY4T6w}#%c4xN$5mgwDHx6hN~?z)=IhTBXK~SEc24M)1EBKMG3t?O~E&0 z-g_U}=h`;a7hh+55rn5^dfvmG|BqZ8X99;vcBXk#&bZ4*vxefpymw}_GUU4Y$(|W? zpvep9D_`i1Xe8=Mn3x9=Ls?q)a8b^9S%(v&t=-F>7Kf6w}X;06+yj|m@72}&bB}|R_BK-9^ zdPqAas_%(y6gsbTygNJ*hnIB{!s}4yliMiqel2R!6qsSR5MYJDaM{cH@po+yvH|XO%Lx@KZamL+ z@Et*GcG=F!e)2_pCu@KCkvrozm*l8QBP8!<7IvLqLhZ>-OxXL%Wp9b0cWrGqW9@jb z8N{8PUnY(N>d0t1NRH!(ZY~A|$#e~WNp{g0A2&I&Cj%y3NzD}Qr?9$mUQ5!vf2NbQ zZmJ{IA$?VvQaY5ND!fh~MNX183@>ZUyupV9Fo@iZPYM(vGlWp1N&2Bjl>O$@d zu{FBh$VfbOyFk)RL8u0HgeDXSAi;0s!r%i2!f!?zSZ8$G1be2S;!O-i-5Ak$K!72z zJ~a3rZbLxUZyQCuJJD2KCqyg5!_D#gZjV1^{%XiS-t8dlo-TlN z9`+n^mDw62-+|h+ggQxGngXaj{RyrWriC(ECcHqJIRmUh+{7FljDwS*J}IV^%0G{& zcO~iUC>XlQEx(ww5Q|G0t8$TCw?hxkbL5Rl*PbNHObh;qBI_(@>qgo=~D(p#{#PA@1EwIOX+=5(qr2Bt9PbQ0ioJ>zn?< zWBorwrkkzO%^a4DsscpP-3znb9RJG#h|)M#J>!b<*p$(m+PW^{K!wXoVsmiqOL6o9 z?@G}2hYsO+&=SN=KuZ7weX$FQwW*63`rSo)ll%*0mH>!28pC-2fnd8XqVX4Yig!2j z8&hkkLtB&`sYh0w-t^(VJX(}LGyaW?;ab-g^yrK0%IOcj^Wys^`yg#fUqh$*pZ;=D z%dtzSInp>6P2nNA*|es)wKb{G z9pp3b?j%!hl}0%BdxF056{Jo-Yc-hrutnm@z6i=$a$PWS^OKyv3jzkDcIEgt4nH2- zqH*C*k@j3cC-lxaB_E28Jsk7O{1U0`RLm7C7$LBL1BNgyx`_i&bbtj!kguoj(_*GN zc#oa(h1G2<15ma*NyyriyDV?gu1yy$G`Z9NgQ>T2W2S(0bIvhr1H4on7w0C=G9$JX zGP@Jc--XoLG5IQ0XJ0F7z3hZd%Kz{D6~qg%))Y zc1K;Vj$Ad5&h6fc3t4xRYE3tH8u3^;O(yL8?w|J2`pB)WvEZG3h2e&jcqXbyPVsDI5Q5v zZhIaWa9CV$;5(SqkvDQ!uee89OPj%lD>(ZD7e$$y0$>I;UT!o)&bX(WmjC>K5Ct4b zQBD}9HR|Y)1hTqW(8~=lt|;Y}0-T0^Ex-dp6*9~OY!C7iSl6^puqvOHjI=zKGBe!4 znvMx8DfCTzA{@;uH=Os>dExI?ZMZH;e}z-_`Sa>vi5c0vveseYLwBMcpNu}{9=&rT zv_*7m0Whj4cpf3^Y;6q}BtKQe_tHv;J5z$@95v9dY<2yA1Ec^d4BnlpG|aKs)4zVWO-8nhFm#iyl!_bJP>hc!4-Bpoj;5rh=wX z`}nbM$Z`V1lrG5TV z7dIyb>jc^<40#x(raw5OC1VPjPu}JZtL?ER3m||xroh67H|vA$#`+0;wJE-2cxPLddJ*<=#b^CRr=N2 zFKb^y!`%!B9^tSi8tpl0lB`8)RvIHk97aWFNoRaxWA_K5lufBFFq7*fZy>iwdTnP_ zM;KG7cN)xw&Dc&UatXa#|3_*31N|p~r&I8D#kI~M61~6ccaCF6+Iwmg)-MVjrxhx3 zW6`P1n!=Py)KcaEs0D`sEGN6`kUZ#_`(dVh$d#!8-tL4>xR{IKSo*O`nZ zDXHa!ojo;5>sPiBRu+}F8e`};KD{@o)2O23X`h>x_UTtlkVTM6bQi4}EImisV%-Dn z08URJBLOBwFy#&H=V4L=yhfJRZ;EI+v66`Ot8PTdik-r%Y)vw+9K2wK{-!!Q`WT~w zlVYAfGZqwwTRUR1SAN8>95tsTD$aM$HH!4qD6d;quPSt_*iA{RWV`cOosq_EKZ#OT zNCV%_lGK$g>@c=h{aVb}0g)MI^I#JZ;0CatdB8Astz%w0Dg6p6^V=oT$OI{QJ0)Bi zhTw_|;H*F&zlZMK_WdlW^EKSRO-H&;TN|IG4)T79e!1?;mp&O+QnJgpl#Os`_tzcR zdo#QJ$a2q=kuTBzyh1|TO$8O<2`5GT87I+gH1Xqi;Z;Q6;DoUhxR^hfI8EV?lAMz34~ns&5e|8Y-W0a#6naNgL>?=G6>o(`mti>AK* zhh=HKk%0%>um|-_5U6jK{~TRY(y(5Lq=A(TD#4n>^ap`LlIjwvhWmP1KT8^B3!~IR zJ>%z=$1y_wHM={gWn_%t`h}f?h5eFAUTG;Pv-=X|@?1&!s75);(pQo4_PKNY3u~0>%2ki`N@XUH=3`VR-i*n;dB^D4w+# zf9o7_`a?yi9I+~-;Nev1o8qzqQ-4RxSN6YX%IPKzDPn~0eYrCF%(pj%~;h<(al6D5Y?cGiNF@Ya(|k`Gk6mH z0}x0MK~@Z?2-z%H2@p$m*Z;2BMY`qvY@fUoCSX2}j5)M!cQW3}u)nlf>e2HXwG&RG zUe?VoynF)biN-3RxRBvIO>UzI!=UA)y)_;zFc5JQ?$jqBp3$J<7f7f;e&UZ>BW+SS zLMnqczz=qa^`|E_oXEwFeF7x(@qu=A1~(Zm`HxXmye%KzvjJH0jJjHHobD<>H9+Ck zwIBZ;28{On12_9inF(U0%6zNgbtg|(XL^FR(;ox%HmD8TD4C2X5#bEom z<{<@$<qxNCaMJ8agypT@o<76n9ZT7_a8zX~!L?6~FCgpn2}{AIE;| z*01SjvABBDD1BCBFE&%a^53OCQf#&%xpn;#Z{@MGpPpiyz27x(YT}3NZ?QME6=JVz zUv9<@1*Fz70t=~OrzAbI$fb*JCV3LZ5J{9smC}fY z-22mkc(5f%e?-s@lt~{yZ7}dBWx)LC{L)-kJivt20SL}Sb%$<13R`lH4Q%VK+_6>K z{-{lP#-@_!^#QjF838cFD$gvfTs~l*rK153h(XG}WFib4PrAAe`7FL9IG*|7| zddlhKm~VVkPtDDBFS4z8*NC#xOzkpT=cBaV4~5bL`;(6(-D`SP+XLN>CaBlJzl7Kt zte_H1{CXPh9VA8kT+>aU`X*4i2EaLR$%E)!LyrP|nWp+8ffah*nylgWxm0VSn|P8@ zJV&sw_qIXF@?5@^%U}o@b7I}1b~35Ps&Gy$EmYjDIiTluI9ViRW|E$8G#1f9mewic zw9>>Iq|zesh6T0G#^TWKFM8lvfYe3w2lsvW4Eg&+h4|?h{;M3fjJ>D>7ORNImC{o196Wk1K2|?=Nz@Yw)_f%=d2` z?%yN9B#|1xTSBa-5}dt-!(!jQ|7sHl!&gD&6*Bc=*R)%CGg`!%-}d5~M1hZGK!gf> zAdhOsCv@Oi%x(rk1XW;s5(RL88y2vllLx5{E=s!O$3{2bqkTHo>eq^tC0A#&t~~J{ z9{>FEZ$RonAICO|Cs?bnimbPvvh6)J>VQ72b7Fte79@eNK?SlcYDP$q4?Mq{Cy@q& zLD+cjZmp)Q5(oK%jqaiqgRg;9a3DK3hb*H;s$A1vJSE5rbn*}Cmn4ohF>O?UJbex3i;rwK?5@E}%B6vLw7a7NJKo_6nMAcVnN0yz9< z+-&;O+r4EdZJri=yLoI-y4YWJld&hU`Oy|&H)wd%8t?Kj*3lz6z28VNDy(~MC`wCBeyHppTia<=PZkNgk zNktvCEht0n{~W;cUFr;S2}nr*BvZ97qbZQa#423Cxl|yjyqwQaL%sL{X#{d4&`H`rwC`^F2bTu+Bxb!oY>LojU~!v6Pyg8mqq@qxNYAL@S>^ ze>OAj;Gh3CrQ(KT1q)Ud`4M0P(njt7G+s#)bHMI2-yL#gppTD|%HVZPCV(vmSTwZu z#8I*U7~cqs0$k@;#GhR2v+NuQfe4^L;}=Mi)4=wgOkapI&~Olt5EeS)v4B)7_wMvU zqL!1ugTN(8e0)JidB|2X5x3qPzH~NR$X^-#-(&P}`Q~{Z%kGoExsCP2MI0=S+)8_w z86xSPE*EB3)&TX4)h%d^52^r5+>TU56=s0UR8|g(vWf*9{#glfEDAcL zXTl*^gR>x8{Qn}lK_`E`x2S`7*(lLEZ27W?;mPMByeCb{rJjE*Stbw0U1XLhMlnfgfLkzV;)efM$&ByNo$y0No>5;aR0&oLN5Drkkx{m_sfLGZI@zmjG7Iy$=bbzvg zjVhH=N7IAWm7y*G$33Bw&(-2nA8sfQToONW*wydN_!6;k)AQ#?J-F8ap-_>S4u!Vz zA*na%XLQGq&3W67>9 z1yOSro5-qcOIhOer>3bOxtSmZ5-JqV*q_l7gkUX9aWkYDg-ZMpKv4G`*+DRXsaiVW zLuU*^GSmO8cFX^jiuszN3;jo$;<~483pG?BA<@p+dZ*!upuTHIL(C5yw`~{c>UQ|M zw13qUVm;yTFLg|23GsA}n50>nx@9ZkGkrAAFc8Ms4eJ6dKhb6bP0LpIv;Z+w7Re&O zfc}If!s~|iyQjzfVT`=qNJ~b%B^W&sG_u`D!ps%H0&ItWt~kI_2E9b?)*qpJs6*** zZY%N{oH;yGfbof0KRLq!&MJMfSS%Q**4t~IJ-};~%92gi`R)RBb05tCc{R{X$jxTE zGW*Kmlc0yxR1OXX&V$#m!wC-S8xKtnB1oB1Cy@UaFKBxTNILRRK$wQ}4d{u3au?Yu zQX(;8m*q^jCH|W=s)L(7(X65M$Cu1h!X6ksQ*swcHEt&!1R9@t_k*HxVj1;VaTuqrdD5Uaup@m2OqG~d zQ@!k_&em*|qZ2ZVUB3+7cvfWpzUvLP3Xi7*4?w&_CRInsUUy)h33vx9esD(tiODO{ zpfIQ!MNpMATUmU!ECO?oAK(|9E0St*Dby`%?NV)8&d$(q9 zga~mD_#90j#2(K zzNi1#Q9S`9erwZk4J3WCY26(%S$inqs9GeC(IG4b!ZWf?{v^lQ5g-pWu{EK?c}FpV zm1j;na!5~e7cF?otK}~GeQqHU+dj4Wmn3Ty*?SC+bV z_jGUhyJKgCzQAQsp@VH%X9g;+c#hOaPq8i)Kt_zV7%1I8IXatb;Jsi)c1NCv82JYt zEQ(cag6v6ieKEAK|L&?t#}b=?MEuwLX?;k8mS?<2(`Lp3ELE66or<3EziZk*ZAVy0 zeDUqfucdOW_@0vwR zilo*p`V@x+ufs?~LXDJVgtwmbl=|zOj|ENlHX>T3hRsIwN=lVf385aENevGG-WACE z;LL!*VU1)6S~~`72^wTnDR6j@-AdL%Y8pI9*^U+X)6~($#ZDN()WDIAt zUD{}CR*jU`2ku46-(@{~cH*b+^YX0N;jwsG`+ng=-rFcA*KD8ypK4P+vfePLXRL%a zLljm7vVIzcL7Gp~`Y|hF`VYC#r3w&TKqZ4g9E(=;U;WhW|DV-wn971B5UowboTV~` zYNT=mi)>$L;4+5B$P?^;*R!A1!%9Eo_c)~_<8}>bjEASKat=>IeKHat(gcx@|F9AK z&XR_DP)rRFFVrAx9Okv?niTbiib4yfoq3*22^bkPBtx4rpg~yX(=ZlI7+F0KLdkOG z)sB~KCZ1fx+F4Cyap5+v8m;4~(gBy06C5VGnsxj39-7^ub8=#i`MeFZEtCrab-+5% zzXdWXkhjq3@>WKd_Oi=eDxQl#a!nb!r;+q+b6tqCp(wl*9Gv@ze%f^sc|q?6z#qci zUhbhe!zwocIl$6C8ICwQXh8o(*?8)nBgLB=vE27yD{hoOUqbs#q5CQuff!wvpE!1= zyY#LsPAP1X$}YJO)>f}#J?<)UAo(u@t)28>+@i&s6jA#zS^42W=wc(}&o@E%5}K!V zAZSBqs-n?Lv-)Vph=GC%cs6a^$Vi2e)6+Yiy!L> zPrm7YA-ZqGJ?g)C1A=D<`b9n_5}tx&i(=`hU88>*MU-G59}Q(gb|&l!?K5zKX;+cJ z(1BnTNGDA+W#y#F5opQuNP6Y`BfS>6(%uWLZG-m$LjNEMm;eIatnftug+WZ!gXj;V zfRhpTOhvB8VJsPOj zfNW{BMINF`r%brxkStD1)&nmS4M62yhjJlk!90-n{GTwSf;v51Is{z&hYJ7#KlGot zp8W)*Qdk?X6`H!`)_%HPdMNA~o?<6i3HKoXEuA<^CqsWKa>#a2QfY6|ir!B~Wva1W z=)hXBmAP#CBuyZP2K|TO0Z&GGSk$S( zFcbUFFo!Y&;ByUREmUske+t^ld0T>+MpEBH=~(OS3v@-UVChbrSHugV-VW*!ChZw5 zT@{7k6Xe&~X}8F{=~q5sB3gVf{)fbzC6?~W;p)2w(kf7!pu4@5-)%r9fO6v{r6}!K z&Dwc+F3{TA+I`B{?pwW2F6Ky_h${?9+R6l@1LHx7=6HrGxV>7Ofdc_^T{ zw`RgvU)7;7Gu& zxUCcNjf^ylCg{^MD*FpQ2JYMS`Gp(BJKyWW*le2wQ$fu{_I1U>O?>y-R+z+{CI-Y! zSLw$Wu+8-0S7h8*R=OhDajSi5gHn&2{V42Jc5WA6&0Tz}y3~g2;GB2U%u*K&u$vZp zxWefmK;}L^DbQ6kmQz_zb@WZ_Ggar|?* z;>b?<(>1_P=0}-CC>q#C7h1Oabsb30_mx#Zx(clG^87E~32|-~Y)`;g-vJQVfQ=%| z2!s&*Mc!9X5Qdb(FZ^!leBXmGdxC;?5-7c6Dp|Z+SG(vg3eGq!*K}3bu3maN&O?fI zVLWu|;hA~Dc*4>%;hE@~bT+r(+d{`ue-hcsSH@OO&yXURJE!xCe#uq4yX85&RhVZn z+LPs9DG|d)-BBqpjopzL`e0Im-TB6K1JyHGgTe|D z>vp>|{AT{={8^TfqK|pg48aj>$E{yhfllE26GcCvg=ov5LHYiPwLZGbd5dN@Y9Y1< z)LPIUkrvo)F%+2xFf#mo(1U4>-=9RG1^9cwu4O2DX#Nf)HNc(Z`E#=t$_1_h-{QcT z4W4l*m0z*e3~7qyni;ykyz}6&pPbv)&0f*%E>-WAjPwS_-9&8lj2VRd2BHXZc>{J& z$Fa^DG)HE%Lu2E?Sl94Jrh$qp?CC)H>*xvb9|$nS)!GAOi*s0 zylJCw(6F=;x>dg2arHW(C~;dLMQS$hsV>&eBXG>8UREvNU#99;Y_6}5;OHAps!aZ{ zP05)yNo$2qs?-SU6~<2Ne70@LnEYGr35r~&W7)ju$}Ef2bwwXL?*Pi!ssVAcuDMj+ z;Ykd%O+$3Nqsqxu+}n#sm9fI&F+*!t-Z;PJCNp+ae7Fwu1V#)w*%#Ps$sH zzSO8Mb2;Q(nnfE}fZ7iC8q(G^gE)pN(q&=lXj~k8zzvr`K~_45{=<$;e-)LD(MKjg z>jHWUwgOb4hu;k2O4$@E`d%u3uGp}(DkBN&k5dNHEK(al`9lmI@^cW{yI@VS0Wt`I z8E(o>h7~cuQDQ1Vhik;Ms~!72CYH+_3fWg=8DC!*v{@X%U>aOE?&_R2JJQ+d-%~nW zFr47B>3Cj4rX)q@MPcb`YvIt0XN=GI(8$O~|LHC_I37C6^E1XA)CSw8)w*uM$%(O5 zuFmBo4oc7Y0%J^QaqbA0vTXOw8!Lixcm5fgog0vkxr#X*=fb5nO?jtA70*#PtjQ~K z%FWLxVpg2}MptxnKxt8Gv&g4ji|t+$tQV-0?FS}JJ&g2vdY4}kpD!kN-d@E$$hg)Z}GxWYw>X+-wbZ!M8fKB zTNzpLQR%$px#TCjt#RpW1>MyAi1I>0ihHNLT-K!FWk;Ddp762z# zdm-k+NK60IZbiZO&yWEV^wCBLFn}e>k5bhU!L_?ZdH)7IN?_jBod8eViWDJ;F8B$o z$rf;139ML9h}hk&KOzNX^2~vXrP{i+@w+!(YFo*tO}D2o-009lmVz>i+s~ zX|qe(UN2rpfB)-Kbf){PV@wjF>rc@w)xYYU(pKh)-QDN?0aLTj_H>cw{>iF%4bnVz zTxI4P)#Y&KB!}rKp}L?a=J#(;9#gYxKUC3mR(0-G&{dmffn%6MiTxMJ=~`DZ3&Sqo zXcF%uv4$ID_?+C!Ti^IZtJby9Z68+j*=oRsdk1EmwWyW8pMLAJRAsuJtB|mS>V6bB zmTZy3E<+#@3G{OqpY`K6BN_|_)OJ)qGKMmEcLXWo4+)4Ftd_I{6zV5?jBT|L)x zTl^(!R|$*z+3h-4s~OsE!nc3<#&LatYYR`iT9}Z09@3{Y%n>C85(3wdIN(wc&oN|s zdHyd!2{S>1mI8-cO>m(VL+*M@GI|1pBI z4$`~Do<`=s{MY&)@dkd2g0h{&_!+b7yB6g*Cng+jFkhf*I_{s$-xWY{t_kLwQB+FokeUHVvmEQj^`V2t_Uro>C(?KP{?rAv;>>_SyMj$=Q}ry8ir%F_{s zhf7?Zojju>p5e$H_xa>~kS1q}+cSu=?4u~n{0MI=ajz2 zfTT+>LQHIbe)mMo-j|uG4?*-CuB3Uzn!uL&mk!e2W^&=v zbQJ^*Y{a*=+6N&9C`%{byMyqO5t8_x)#8=frgzgDEWS^d@v^Q)ZM?XFqlk0g1?n+I zenOXxzy)eRqcf(;RXn|Cdt4s%pVR5ez4^P~SD(nA6=Iul+qNf#4y+D7E0uck@1aln zY4Nzb%P)8Ai&VHjw=AAMd9*5Zo&lGy7_&V=p+ZwW?lU!Lc#+a-FugQ{*0Cx*)Gs|a+c#v zar5~kvHcqw&bWnNnMhbrfwOU1Q%snx-ebA)Ugtv3Om*+l(6_Q8Uct%qlxDRHM0xWu zRqvWN?@aPVhh0R!#$8HkP_Z%NQcLPmfv!BrUo=}UIEe|eit&+tdU$ow*XI7!y~P{K0=#s8}9XU~IEr7i!XZ ze%v+nj-hI9&Ih%Zs#s8-+YnGglb97n+0RKw&67S`t# zd$|RUpLFiNA|KB%{*QRpyMP^gWwk1-)@IcXpSHIjEnBp* z)Vkiuvru>682|O1NA+`QjkKY9#T-#TyDMGejMg=>U2y;0l^1-Jk^?ykyR+TG+llvX zO+O6Rd8w$Hrgf-cZu2Z|z-9XiH=iDz4+EonS-)0{lg3f9+to=YjfabeM=EV*Mn9K% zJ(n)FJ_(pD%+&C3>@o61zj^bIVv+CnC|A}j{5m$7m6^wRhe5dk)$h}(qpt_Ja{7Plq*N8ET$T&EWUR~1pOYU$(rBS zK5I{z-*kt#0VzAVMZtUK zbmXy%J#bh);l8JQY_45)kkLTGcFftlEnf=_W3D<3;tM(Kp$vDdDqpzJpCHYpmOg#u zQ5Cz|^pE#?C#~XY**MSIenL2lgX;Ws<^5A0Sq`z~UTZA9)v%Dv_U|928#=y;9;zaE z>+a*$us&Ac$+2X7UP70hJyY*Y7ZIRC`rT9Lvz~w?8>oQ`UAh#7jk0L zzF75!TF0E8WsW;#C^MS9du5N#{K80!yk3%F41d1Dk?ODWf^LI3lJM=>6xEViwRUGS z$0bp3sfSmpOz+w48lDp_s6}ApDBAL6*T}bB>Xe%;n{s|me3<);?(-=`$j3vD4<&B? zq+Y-pzOR5e57&+$VzQ0$j8hsWS-!h5d`m}kS0mS*7xEqdU6+AwIZ9HYU0JC=&<^GG ze^h;SSX5#6?aPIC`*+J#e;=d zfVCub?k!G{`DTRG*Y|9q%q;sA98nARTj zhMQ&ps{kV*?%kr>I9US^e%pV{%sjC3v)L(f{6hvtN-or^S>|jeNX1&bp|ER!+)9 z)Y#8<@j9w!xt)&Um0Efnzw*b_mC@qT{118p&$rN7p`LPk1+bel^ACf;ob|ii&DAqf znsyl74lT2qZZx;T#YgCD26~fW{?{M&*Cf;5G{zS(3>M>DpTE}DAQ&!@P&}a_A4Fi@ zrL|Qlm&TagNe$Qo2eq2g5+N8zGl*24&E`S)yZv+1CW%Dt*@fs@QTZ5 z&yxDA-wP~HRi*fQK7akTsBMv_M<0FCWA5dFaiS^Zv_7yf_~%>Z=!=V=PovBmgSSO9 z3w19j=@2dWnfP!c)r$aG>IO=V9?t8vPdO89bjR6IlAI%T_3K0Ydr@M60K9FDf_2>M zSq+=Xr47ekOUZH^o5gBpNvk>|CO7jgARH@I&)>4AtWvrSrs7Cx9#u)aALTsu!Cf>) z3jWg&0VD&=EeihU2%Vgaaid&||FHuDo4Xb8IRm67KrjDu>H~Dh10?#V0kb18sOiiA zbLxXW1!RUgT4Y`*3Rp06SUgthQHx1c@X+Gg*&T!xs zs6C2+E6P0i-{rKD=U85BG5Z=JYy5FjxKR97c7W#YMJ=$j!Q%x|w0m{Kw;$dBYqqsK z<0YW#acO%;qBNuDOftR&%nNnT624ph9(Z52+s-ThA>f2l)g=s_8{Q-WU>QA*y5{8f z`pH_VXQZknJ~`}g|LpABQx{*`b+pqPo2V&gRZZXX$v)db=t`2^J>3LOHlG^DEL059!nQ9s{on=q=L~`x}ICp9~Aco~s%D#8uzUUDgu-mi$xy;&r&CxT! zvFsN+(cXs&L*w&@w)!U>StlJWtGX{YN%T1L8Unzi&arANDRkq5-0>3ue+{L~br+z8 z`;e~xSGuO>@%bNS@%6ajfKO}wLGuDkS^3u8q7)BPabK5#+0g5FhX;9-^?~nC z2ac)0p(|cAvLGXMIY##y5^Wa{Pw&7Dl>HVdDNgqId}7ItMtH8!RG#>T8g{46^-V&hv_Vf zrW_WccR21B`Lo(^YOk>{Gk?xHPocsVeg(&^;%8{o*{rQ!vZ-%jKRrbiq=_0xUzD^Y zQ)F44&XWAvE$W7kJ9^zF0_R9y3rKh^-TPmiEzgb*d#%V^m^c;O(_P{D?%&sWt*klf z+Aem`UDehfBD(lmhg!j{Jb#Wd*urZMFhrTqwLxevvDpaZu!&WiIodCt>AS(%$W=Tt zRekxL_G{TLj=F0q(sr)jy>}R*G5XB5SSxwqku|jHT`>MT?Rh@TXT*Q#E;^y{^p_vr zo8GxNXGY;2Glg_$7?&5DsA=@C7PUU&A^~Q>4#G8`$C;F+CyV7t+iv7PlIg#G8;kuF zgmS^0sPBS(8#okRh&}o)ccSSpo7l%M!uBKVZ>LKPyB|IG{aS|4vuR8*FrDmuJh_8{ z>|;IrqPa4=c3b0dtphh-C2D8r{8RrNLQX9fo-F%`qAPl1-uo(iWKk?vYT@3-yYzmP zQS#cOQQP~vh?7%@#N@Zk3fGu^ZWEJa0&#jXn*E5goC1`pd*{kjb(*)WY0z9Qnazj$ zXQX!|*OueP_;26kwSzcERIrUSQW_P>7wX`@84Ujv9v{PK@)ZPI#03ikF&y{>!5twL zLJ$}Uh4@H_gFgnsOb#r7N2dvb;xT}oQJ|@D@l~LJR(Xf4%he`}+zp&gFKR1p%iIm# zPB=CZILCMw0s`mgulk`NGE81?EHuUJchFSB9YNnJ&m#Y} z%|X>dDfbk(W)wta)L1|A1#2PwVwPi10#QIbBXdEi%tvZJQuV^+?By+e2&9cKX@a)- zPnKm#yD!uQjre1;efi~t$cU{ia>vBGcu+osn^19_MOJU$C{rPh-+6v)=Y{a!WT|^f zh|=#yHhld>tfRHS{!MT9hgNs84cXJTr|l?c@BDc+83W|lELM@4vX8|zk+g4mBh(n)O`e_2x%=V&Rk8`?8QRoY6}ZK-kLZ1aal&={55rfFj%wkNcgr-UX0$+?673Xkg_B6z#wZt>xk)jJ5=Eid}n%8{~|2yoiEmZx|!NKxWI_c!I#1 zAr9>r$jT50;AN61NJ}7c(2y03P%#iJIXwOe2*%b;SvPi!zS@fT`EWtA+FCv4ViXQX zU5zn%xP6u^I1mNtNqk=+%6PZf^Sg9SU%PGgqcrfZI8~CuH0eLIX0Srwks2XL6n32T zrX~!*Zmb{=tUjjNY7|uxj;b+H=F-IKT^aOj?{z*#WiR}m| zP*JkveWS^1WYGBXl81DN`}l7EUeu9$;0^G;3+%c&ULf&9Mm@gD-xQ6G-83o4dW&v~ zA(_vgD{QKLpQiY)7r;NhfpUVlsDZgOsA|y5qUP7n0L(QXeTOzAIVM3%Sx`l{~PHlwAvfb>x{ftZ9a@HfVi0{gmUb|QC zCggTg8~_VP;!~$(0gE%g z`vZjJV}hn)L4gcVh!ubbf$;_9jMdNsO9s;*0t36=Fws#H9myw!kU~N7fe}5D2q{$L z2pF-5=)3^d9l!PvJBa^%{z>z-Y-1M2!-D6afcd2!F1(}=%a};8$(cS$-O;di`u$bU zOOOm{w{y<_zvNuqPjI`0MwjpuXHdSYGn;1<@NCjx;v{=e)mdVL2(hL9_G=QScu=jq zcl};zU%E0|2jQLBXirNJbDX7Pl?7k){5+F5cJ$@3RQw|r2K^=~p!GrX?eJy5YVW6W z`PXUXlyY)TzaVTKy}f^UA%o6A6K}~zZJy2%n{--k&3`z}{;1rv7n;SHz;t&_h9aCW zfA?*M*gW-=T6mWaKkKjXOlA&x;W?H``{hWm;l`_L09F#O|+fxxmj{0=w(fz0)+B?tK!?pxfzL)p?5P_rZzTvI_7s^Ne zxa)Z8FP;};DsvN>X4ck}#n0^2FtU^dSZheAY4Jt7DM;QQMF&5n;8~HN8)t5m;0o|C zm$PMI4|B;KT)y=Ev&WqGCe)njM(aEnk7DwZgW%&4D-$5W%Qi~m)p(Ftac_shc{{zf zLg769GiE{X3L+>~r?dkEGGzRBHIWB$^o#)!4GQ;B>m-zE-xH}8Tz~^ z8dwbYIgtAK!OkG45G@@LCW?s_sYC&hKyq+!HwA&6{U2_k1A!4T00jg-q0~nZ)Zy#R zR6+NqyOQpW9Cu_oujaicLVhZXi}3a(khp)b%ie&#pdT+TqLQrRy_-fB-YdDkCO3DP z$915l<{$|Be&a?(^;ZZq+__!$qbgC$BGlBLEPN?M>Q^3|9c5R#Pdx;A6fN=U9><3i zw>`OR#L;ZaVHH(rBL=sIDdtrp>&>>2jUf#xB*h)LR^AssP_qiF{#I=&82tr~eVi|Z}E1PRWZ1OoOYS3WmjmWw) z>ABBoW*~FpkGXMuNY3}Ts`B1QwKq-+PQNGaAiSJA@}(wpPaIji;MW2ev|h033Vq{C zRUp*WmJF$G#_#3Bdb$#Fc;%9O4oWu%X}z6E3?9oo3JV{=N*_G9nc?mn z%8h5IpEIxfs|F|O#xau6i){3HU)^B%`-k6guEK(*5<%6xfjDR+0E+o)yAT7|iyL7U zJ|}+i(|J;^&&(SREUbL$q1gNA`vsC!iLx$d%|yoxdL$oE?Sa<^!TH)Q3d7^^yq&QS z56A(3wTgZvkAS;CPrjlYj6MXu9E3LW_Hn$V|GvQ}y(phS6rY8DjHV7e1ZO-2NX}s?^4VB2*xif`#D+ZZ^I{Agp4EjnNz8#?>yqA}y8)L!R3;ZunemS#EO^F?}r&>0=Z{S37&-68=eHe7box=4G{*XD<#w)5@ zF!eeixk0DTzH`ZfcSy4nppaV{oGtv#iX2{i(eJT5c|4Q=!l;tzw#E-rZ^jx5T>4o> z;X+z!Y80I4bBm%fja=4Hn2M2+-GE&oIQ@G)^7%Hns1vLNvw_l#ig*B6why?<%OcmU z#{}5E;qg?Rl%dF*Q6aDd`6B9yvdf{B6<-#lKbP#2BO2I1wvmniAtY+g`R+>!;qeQT zkn8P1FrBr7C+e{TXw^?U7*4t>aq{zmq|<9V6qfR@g6A_XroC6CSirAj8{P~GP=n-* zkxppZI=SemKe}dvL01EM!4^fS1M$39ClrpPvw7@kocnGha+?G%!w2@x&jqHhqRgWcnjUfNxtk|x1PBY zdD8l{=Cj@Pp&e$XP?cP7(6P@GMlpzqMEI64b8aT0*|%ajKyCdIk~oLV3%*|NspRNO zf56YmREnUWT-h&RHn)D8J2p>j@cvf_%Yn=8*zz*&W84`z7U(6$x*9(sXXtK;%N^fR z4ZH4(4a(}#c$dHLcs3`5cPn1@07o!x#?K>XV;%1$vAL;S4j%nEqwe-E zY%y`;z5WK#?nh)r#T8-dGi4>;Kp?2De^9@ibcdXkHZL^kssA+%h_o9X(UJuTv8&Rp z5P1xJweZV;7O1ZeMVZIF4Fh>dSnENy>Sv$Jf{8FQ-mM2Aha1J4-9=^PZ8k*Q*o402{54$&jvlT8;<{=CXl9ps8;kxNTDMu0>6qcs+YL^P7;hM>)I(j(0KGB++M(( zhI1Q_r6<5P4_Va=V^bH=SwEFj&2%c-O!G++%&;G*JK?bQlyI`f#oc$_~oHm1B+@ny0dI0Vq(ucX~XPhS#uwDXQJLO=2bW<8_F z*@}By-~yVC)1tvC{lmH%)%pke6pvBZ>aJC#++mZ4Qh_m5ZV@s^N)dBjrL3ZIf&C+y z5$Z?Uw%4)TN;i(N)a*oX*>!?VTlu)RDyO+gUXDT@8SsqPj&GgIkfU2By^@5)CyZbR z>Wup?d8#}c_4v8S^;W{wHve2AUHn`~hNH;=71HkZhFO8$r@~}@AN%?_csxU;q$1Lk zn?ih;xu_bQVZ^W0hc=*9fIC|use#xIZOBNcp|mLe_mlEgFrk0MecNK>y)PHedW_nx z8&lnVOhKs~Utre<=R%g*S_pqxZ)X&L<6OjnL>)Piyv3B9H6w|iD5f15UNawFpx<6# zLC~U80r6d~&sq|-roo%)bl1tM>WEmLJ}$5T(SO>?5u(aaht@}{#g`PTIcmQf>}@vO zcHJFBv!X}VW!lAKUMX0O@1>fGXM8U_84z!2Tb42Kq}khd5;9wSSG1ImVI`@fUgXB6 zn`!d0XTgq3WNIuf+IO(SRH;vI-ko_6XjzOBE0jSm)YYgnL{8r&1*@Jm=;|3N&y?G7 z{jOj->A@X*ZYr0)!RBs}kvvjtwCk=-pA{`Kd5ld)_nL;|U0wEh)#OC>R&%|L58t6# zNv?lc!{c4^(Vw@calYc!vZKK_4M$k>zCcd}M%DjOf4vdP^DLUA)9IUE(;%FM9K2V2=ZcbZ4 zxhrO`%WKQs)8x4|T?zX}4{gBYs^GmsX2vH5Ue# zM$-(W`^N}Bv3u#YjcK-GqG8jhKQX@Ry3pR33N_bLU!)}0-fu%yO?_YZeSktcel(vq zfB4aPZd%g2XZ;L!w$h6JPPzRrQ;PO);;YAxDOOu+$89M)FJ{nSWDpqcY88#XLfHzB zg?^ib#TFAmQLZLGxMeGqZGhmCeqCr+EO^XDo$yY|mh%pHs%?L_T-$_wp1;AVW(ODD zgQ2MR*+0;4do$%>I7Tr42u zpK6&M?TkhuEOS~Ot-y&X?pEEeK#qB+_!x*^^g)QyOp zrD#lGOy9a%)|&ada0xaclHZQ zsL|X>0V8)V!I)Ryh4yrHDY7&M3x_1m-2JQ4gq3-$p6MxUAc+9HZJ9BdwJFn zH)S~b149I_j(aavu(g>6Hq2-OP(M`Nlw1g+NZY+tC4z^YF&vHgH7D3QipEJ$SI1a$ z%SWu_X>o-54man83a1&&Kz*l)a%=!8DV`@h5qu_oWns&ATP^(6vubQ?MbCJlL;w5W z?Brbo(1}Vgch8Ukh!hmCu(SWR(<#{1N98n3{|mR4VOQdvpKCZW{LXd+GO2xUL5;-L8O5 zV4(4Yl$%3dPYG7VS2X1l{^&Vpj%d40-q^)3>;1C?-Hj7YFf&ZLCh>}l;ToVZ#$9j_ z%Gc7}Gt}L1><4Njl486xw=hl(D}%#Mn0Ffms>`%m2_`tcPN>U;p?B3W1V%RO=UD9Z zny(*l>K)@UScsO6{igiB>N)l)1X>R)R-`oHD+fE+%`yJ`j;8=>>wrBm9a^b;vmRO=6=?*jW!{w-~VyJS)7 z2%MuhGhffDO-oHNUA(kXX;t2`Mi<5Z6_G2 z-K|t?Mj_58iRb~&Kv-)h;&t!-*XfmnzY=QOS=eV{`xBkp^E*9vi;XOM@lHA5bpJU~ z!^(FS#sHX0SO8hm-|3t#*WpgbOfyAd`h!+NelU|PSo10x{kU*OACDEn|K&^$zT&mI z*lq#gW#>>pbtLztU0211Ihah;YuJnUudN`N^D4--#A^z)sHhaS$Wk(4F{4&5k{JU) zD$kVAV1ZNv)4u%;%&P_f1I)HoSR;h25J$ao?YpT7lnfy-a3jg?3s!s$F%Ka4`3vBb zjit!f(S{iVcgvw#G<>p-cHFxuvflKe!}B`=t|RdD#xnyEYkh@LB{_@@DoMw;#7^%? zQZnKUSK^dxM46ryutC;uW?tKB9NTV-I)|Y7xV6X(itPT zyg*bJZ)%Uo3Z?N==!7>%I3-PwSEOUsiahPE9oZ|S8{1}*@&QSny`UI((R)SK4JZB= zb{e2SH8;bG)nN1TGRLb-$1g^%K{t;f9#Fb>?@I0>;_Qa@OD23{)OX5u7oeo#^vQj} z*U7#M?b^z%2D`hFTjIc_rk}Ble-6O)a3Rb<59k=YxPrRvoMm4y^;}6c&9PuJ4fxh` z>$U%wjBQjH04@~bX@+tJ=lY44g9Kb6%Xt7cDJ^SkHa#3Coe&GXq9AUkg*c*IVf+{s zu6=dUQIlPmu1%Xvb-;Q3ftN0(;#<;l#e&EDpiA)Qz)a0(fyZqI8r6w*nQjj$*dHC? zZ<|~9QwVjWdt z9n#ICcfcaAomORj0QdmT4YX3-z3#0uHu$0U;GR|JlSt4m^r7jb6JATuN~;I&n%c); zVB+$Ib`M%gitW!A9U7}4yKcL`UDw=!F}qHlF}N#p<5H+fh4z2OZM)p+Ksw%1o~jVT z8p7TQTXk1`GA$$tN8)O*kaA%=j~@I`1+S&PxSMjWzH{}=Ab!z&BmRc!hPdNiYR9x$ zTQdU4#DB8u+#WIG4K3ie=qYpQ@Kq^ZS^N0 znQBMWW`2H8@w%Ipd*DUm*!HJO(*|kt+^Wa3vJ~RXAlN^k0CxmPGf<9mS;V8dgB6~t zw7cyRWhSHjAI0%Nz3oFDUuA^xPbZvyVT1xDa`Opspk!zO8PzE*%-!zQ4#s6I_PqQ2DTxF zIsT6lQ^5ebNtTcP(32=f5L6#;Y=?Lf7ay=k28Ar9opg?ZCn#ZChx z1^2GdIZgH<`(X})*{_Zl9T8Y}r?^2!CXOnP!pv{hrN6r+d<>2t)I;)gxl*(0i>=1+ zW=8riUc5Rorv5umrczlgm-|i;%}EcFmf@oxIZ%&(*qYzGeqh-YWT^msB+Qy2Y_KzD zL%&binz_F&ArJoTT7q@Ta5rN#@RXT!(4J|wS}O*9k;@}&H{_SMB+@BhDBGiQL0dBt zDN|Q5QZ|{R06wDzCq zaA@e!PLoR;dzk(CEZUKx)!o|mL5ZFS)b!U|dq!0MW;ul5Z{hzOPqjrzdiN!HPlCD}jmsun(wc>bRs_dakpm1FTI5)8k6Hh+$Xz` zmY@+w{=%gVJ*$B>*=e?Qi=U*K@VC!{uN7N-TM+&thi}Z4UgwzDE}Qt&Tf0k}$MFLa zJy{T(%8DD{y7bnXhOl55on90V8WDg?4ccFK@%c;TY{Ipo9QyB+v6r6BOmCE!tA!v} zfG>cdStSH^rFd+xNB)A_xcj!Pj z6w|xJWyW=q*{c16jLX>$2}&;?-GvZmVLCKL|8zl zB3E~t$qG0Ec334@_!cjy^C~>F_Oi;kCKiAA72JW|;P`x#XD@F1^j4Yl>Ncl_X)zke zW`$5iMPcXQFSO`+6W$dGn(^?nyiu+$)hdkEWPk-|R|fYTIPEXD zUeJN6ZGWbRliO5?YFvrT(0M<^kkRg+GfAN51%QlfBQuao40o|t@4(s=S#hHUY9=C9 z;yNl5$nxSJXi7dGO!wlR+IbVxLvXMp1K@I3`SWn&bk_LM;Rax z)PzBEQr>lc9Tg5XM+tKOI+LY1g%?S<96#Oni`Ym;E2`5IuuoWG#8hd|wBDwL^Kbkr z?ugnP%)lNZuh|_ZOTwhh;|bd;_F@7|2OUz`56P=5^-2#Z2lUFD%>?l~rZ)2GIC%3b zTOjc~IWg>nwqvp|>%+)}?lXZ%>8v52lR)a4ey0vLm(PsSPu2p&-Xg22_XjN$5Rd(V zQ+pXfFBN2UwJjZ*b!9;-%vMy7A`kRG zpaxYkNxvqR{(Sr5`>@Yrb&qF!q6(=-VRn^o`~8*_nmSfs{%knwVgT1_-W#XvPy%pN z8y|=J8+a_axe)V~0S!vDWG!+zCKQ*?bbB4bG7LZ2fic&VCDRW)J+R9ebej>c?D2tPcLgVLj#QNk?nqXT=@Tx?MR{QR|~= z9Tz8z=~>Y^jPWblAI_X97JGkR3DFtsL|?7Opn#hIR}eblqH>V!q_DKe|BO}yBWnS! z^NTlvi|5SUF_8Mu#5vDQ4Zq$#H#-zLv{csV1wzKe*4vX@xJCX}S=^@maA1n&nBn{l z0BVQEmU;P~!jn9Hd{)KxaL;f1MB^lGydf;_cmF3w%E;2cq%tBGRYQtSTTcsF9R5gr zK{Z3k_`?v^@Kcys5I>F8fld${`xsE5n}9|ZO4^VCYeP1P{N zN99W15t~f~Y3e!y|IjFEHh`0cNw21m-GCLKT;87llKoj(9a9m6aj|2-skJiY>8tjq zNG68?N!*5cfmnUpYFS%mj;iAI<36~jJuJ$1t)%#B4>gu9^b71N& z?u!TJW|Xn!^Uqjh)D0BfGw0yd=VP?)L8GB!12dkYO#iPAP{1P+>L7u)DiMqXU|3=O zc`m9uGCrW$zOKqzfNha8kgda)&}3pe>(Jh->JJTLAqg z&sTsw%s8Ls-|7x2y2F%DCrftMb8OJ?vUSLiI$Esfk&(k2fyyT+8EUDJ10le^ks|D@WdtDp3_B%1igp@KDe+`B>0CwdPAs)01pxjqD0MLGU=wxV%NENbPlV{DJNPxvJ>X;9~+vtzGY zj&44D&h?bqVE$;v%K_smwhSnIoFLy`O0Rzz%h7|k8-60N_Ckj{2d3dyn70i#sX}eBvGAjQlf7sPr8}LZ{+-1b^d%hX9h7H~4V1JYXPGZ*o_UBzV zBktcJX&@1L_0Saq^aBes&j^=$vm3oZ?_1-2Q-%l{H*TLh=|8gF8D28B?{kUE(znav zH)7PO1W#~UZ&SKoh9G%+&dLfoJGxWKbs+sVJx^HU@RKm*)S;L^nL1Eb8T2~4`~#f2 z%CMWnF~?mmz?iRl;tVtHm_Qf37HiJZi_cJTw$898pS^PWjxXvovAUptdgPC_iq)6T z0+5fdvy&157_$iOS-$?Z>`4318(2hU0{;90pqUvI*C0QMwA zH)*C=TmkM%hcLI8RbO|4U9uka5K#rloR*1|sTfgpn z-V4c*znLLD9-Ymp$jc|uUtpRXuT(S@fpT=4cik?=V_>RgR1_zriC+aYrx9{ixI6iA zQjgARIi~%{@t^cl%RUcg`ZPe90ufAD`j$1M_EMl%)gJcS`ES27QKuJo|BUj3_sz>9 ze*+MDf{@%Wjv8L@v`VSh-tU+WoJ;GW8VzmxV6;O)-G2_$$6%m1z} zNMgpttb{NAO0BNF`VYJcKs^p*lf4-+AjZ^yGI0IE`5w0(+=v@TfTI_OWCsg^Mp0=o zyid<#cM`RX3BvV%lYRXry6{c37Q!o@P*N7K^_BZEkaVC{wQo|Nt`Yv8{~D1hHxfVK zf)PQm=7TS~V`VXWR?KK<>%)@k{OTo=7{^*K@xZ~ex^~M?+0c-qvR8(^b<&xu=)t4Y z^z|vY0Y*3T2g#Z3QU4p< zj&3gQH+M*Wkfceopv5Jc3a?b;(~1qZn2Ani^UBM;`EN9s9;CuBh!-9l$hRiGMKAbTF~CE}(9NB1PxK+#;1I*) zz2U&i02=m}+>>{8DTGcBO0_uQx>uy?jp!6}F6#2`i$;toeCUm|+sp}aTZ7^~fh z=;oa1=4$rB^X$pKFklY|K{^g;GD3BuU?rr`DQMlUNWs)hl>Ten;6~Ma944IwgBtZm zVRyIiU`ZjmHTIDw^CatTjd)SyO^C~)ra$MS<_}PsEwlBLJxt!t5p7evPZB*j_(Vv5 zXOY;~nv5L|9pWw{VAAqZ{r#eb?E-b`J*MjT0GgAvjDrgsu=6{e0@*|WSEMttuul-VcBf@YgCO_MtAsm>0v;vuF*q_;I!8uVon+dg=UC-&=VP@Q-#TmtgkYt(Gg10dk3eS^0K-i13DN1edWx?+AZLJdy8EfY z%NW2xbj3~DtiDxR7bUuYUpH#krFEONlEM?Nj#U7!|l z{uu&ErSO#Ck+*8oRbHe>p1AVjskc?21p(UVgWC>BVekLhq7grxF^<0y@a2&^CoNDA z4YKq|)E|s1%O9bDa8Ip^R=E2kD|UcN|Il0k9Y)mW0ibRtO4QtVvocpa29t&pgolOi zt$}Z8hOL&-%O!2)Q5kF2{stR?r<*or&{7r9_GD8)IC>2^Iwl1`i-twh#HJ5u z*!k278X_I!i`yY#e^oi^V}I>ahOb~Iew;J>6|~!wjW>7DJ*^R2=fY*06*q1yiG8q7F`w{n9NPr`I%@(Q zx-BcnDm-%M+Umr+3QZGjPXIyH@UoWQpn~}U(E=1e)(sm(x%utyTgur!k3M014f8S( zh{Ml{o}Ls)zMuzc`%Q`-5=+dLIB@i@=|m3ale{B>Nv^8%MEOIt<@0YW02Iwxa9NfCfWX=V6^}In*KE{9VRYa z=v_SbkZw2}@Z(XEzznt{i310QOM%it7tC7b@9P-p| z>UJ*gIRql9FFdC2_&#?cJ4k)`O(%UA`Pzkr+N+2!7jKznpQ@Br_Lgm6p<*u`0A*~c z+5dfKyP?;6N@kw;GC2Bm(f1H;Zzp6G$lJ-0>&saet@91~LDivxH@)oizSNYGlPg!v zHpNQL6=73c{d)UQp+*Nq#b`gXzc0ncZzW~?|Htn!`26!Akgej;Bade};g-2{GI)>X zX{f}H2>y#22s>Z=cnz48&g#7iBLpX7TCbj7>hcDI!?!5(D5kzt{H)Ot5N!lz543Z> zY|7W1d%l@w6OF!Yfr!Rek8O`*IAP7V)Q|7+1i`xipF>cG!$%TczxE^}fgiCc9r}-# zr#DIqK!Cr14FYQfl%`2mP||2#5o2x_X80@oYS-jrdp`PK-v%!H^j>r6-VRkM%3H90 zSL8Mf|0Y49leZnh(O>0236$uxu)7IAKxO**mGauP zP`yczx7QnVsty;}BKBxiq$>Y<$7pCa1Wb`pncY3~D{y5?{}C^6t5LSh4(b7C#@sw? z)vP*PA14o`f z9Xs52en-tNK#MP-YAA5+jaZ)Z_TlduMlXDYKjC^2r1zp1LF@VY#m2R_l_$p(g=>Ai zVD-b~fT5az=lqKw8f}J-G(H(uz&h+fEtQ6fk9t=y=)DWV= zy`jZaC0va?DX79zCEKs=B%~lnV5qy4O#{b%4qEyjk~Zl)>GY6|B^Dt+{Yyj zq&N;8C`tOo9rG$x#SbBVVE>eTIN8xv+l;WQJ^x7t%4#D{VWL*p7hKUJnr9dcRJn`Wtp?1UFzGoX>1D0fD85P5nI|tOK1Sd_7U$*6N!e z@l=XEQpMAY@{rRD=3~9dP~EV=A^p?3UN%d4?xPPSFie8CvQ!2MLFo2p8D74lVdmXr zcb9gB)R!K7s_4?8Q4-ykBR5AxElU22RW z3TJAOX&W7QaEa8UlD?o-(C&1!i9wQ;r+!05NlS4nyRFjf3Dz2V{o_+CK`?8KhshAa z^(GVqf6@ENp8Ch)w7bMGYCx|G28Mh9GqD%Wu?gs5%?GikCyCVD2EaAZkd}@o0roZc zYKk;`Hfu`OEY|%M^|vN0ho)8q4|JojhO)^H&2FTgPf~96#HC}QRg0FINMYQoH;DzR z{x991x8N-OiWlQZuPHcm5{{VS#Dt7%+D@kU6` zO+HRy;nE(AM&bo!@e7qy^kT^bO&}zI>Gff;|im&a`N>ZTR3_*wabd?*y@v zIFFNmMtl^C`FQI^C3@l_G4y!d+@}BZx_WKmqeI*0kdJN{UBj89?Q_jqv_>2rqG3Bi ziS~SNe+K8?*kjU5o&wwuKVPn$_pfuWZzD~zxo)O<1){asLrewB<-HwB_tY{w23dI5 z&pppA9*+tmHq*!L zJ`5Ir++(nw;7N$c!kE8W0Lt^fi9{WS8-0*@5HiNRVJsqd>KXLDQGmBk^5thR^Z(;w zBvQuiCo{aye@%NI*KvB^)s=W#Ow0yMX5YKx-bwk;T*cDQ_3R8#u2)WbeZ3(+E@bcb zU=D7?Bs^j@n6~KNw*Y~41UTDz=g9qu4RpKY=|2UTuwYsJ(!elG`dcu%0y6>d?JcRC zQ}H2@EZC)H(ydCcR%?%1fl^E^yP3X9-5bQqbQk7vkJaG9m((%@sl`q1VfKeKX>09L zDQwO&&Sv+8g#MDkA3mH1g!&(0e&f~0fy$;YsUH~!Y-yMIeK#Fqh5{dbb^^l~YZi40 zS)TeLfAs50&5ziLJ+j)!DaWWgWz({AHbKO3XV{MvBljYMh@}NVlM}p#%*G`Eu)1KZ z;T%Iy;L@UhNjCnmb?uu(#g(|8NF8E5Ms3EofJdbU!c+a$>6uvLb>1jO0uqDyeAwD}jcO}cDduaHzk&K+30CxhgM9__ zS#2us>PYOmmHe8hn2s+rzfb2s*xG6sQ5Ylfd!4xB3ksC38%B&I?l3e^ezN|oBQOGP z%yyM=`U_X6a9Ct4K3r*mZ)1V*ZfE9FNj1a%Tm5r@ud)A@WCR3Hw|cRTYQKAP3K!mf z$z74uxD7S466%$dQY1^+hiPKMbw9j^nN`5A_-|k}z>V9{^eJOPBYv?znJ@QIB2#Av zdV@0Hoj}oC##tp~sFxTAM4GHyk`@3Oe`O3<+*gIb@K3H|x%*DYVh}erJ>EYP7Tj_u z*37@C9gn`Iz1*0NyACFH|M3oUUA0-pL;$-2U}5;4;iX;KNSB$Cr3}9~`Q2_9c#P#l z9t6n|Hh2k;t0eOi%c3RaSqx_0hM4vNPHa^Bk$~`r znWPw|j_z$dIMe9@pTg!>dB^w9{1RvA`!(X$-hN6BUigR}BNV~Rx+z+MJBjvho`7Lr z;{UMq-r-co|NHnsoa_~n>_}z^*;^u!vS)bKPbo%6DaroWtD#3j(&w*NF^wl|O~2TevDL;`vM&^;eP zOQLmbZUk~b_J)NI*6MqZizE_ckE{=?8iT|&uab&ujlZviz`RLAujkdtO{`oxfmq=k zipEv|6Z>QxuC1T*ELgvT!t%)TOW;x~VC8-eiK@R+e@mG!r;9HdbT7jD?16ycT$Pl>LCUd9%%hb>ZM*=B1+=^Pd~u$oIM~O z{x$13h6M%Ga+>CFlqkU#Bh)|+auxD{}WJyVx7G-*V8%{fxcLBbk$koV+9^BSxd zrIOP-(lA%al*_OrD(^K1PyF(uF(KZsww=RczeA>*Xu&-sBpN7D$picug+sZ&tPC|G zID18mb3cmE!Ei_*6bx6Sp4_w=jmI|{sND}IFF9Rq1!TdcpFxE-H_urDp$FL7_F!)* z$tmB-gUk)iJg$&!zD}A*c)D&2E4VYT&4>P6Z`MGKFoXf?g7S{V&N0nB4;uM_FHqh#k~Ayi9}PxgJ4eC zXSp>RjgPOO??`!PfHU#dhUq)fSbLc_euQJ@L}DWN0!H!HeG)7+p&Pzo(n-zHi+`=P{PR0M(tlG3=!Skml3 zN9wuqc(|DFG3=LV<&_nsW+B~?qr4BvyQ=T8HkTN9Ev6~l7WQVcG6jV3cjkP1smO1~ zrAMICR+M>K{z^s(3vpZQwIH@es%gyQUODFqY_L&e>l;A*;>Q=jeQqneJ(exW&{VNrbAB$EtanWEiq4B z|N1I_;=w~jB?t~8Y8!n(yp$gHCx~>*Js*v{BiRX7<|==GSNVwKbLVbDIWjPPlNf)x zeWg))YEf@RJyTf|IRUpUP{|W3{HK%#LJAxpt86==1Xt>{U-aQ5&QP|*4YDJ@^!a-~ zS@GA64iwkz2uiYPy+0hnxzZ@e=3?_bO~3Dyt51OS;$XTP?T)*%%=vG~;qnheUrnFI z)aG1&4fb#TwiKgv(rnc8PCwhWK@dZUxgMea3zO+f07KtY;KIW9VNzd1)@|$_ueZCW zx8+vigY2F9SWt_e!4=<}S|ODn%S1IS-wobKcf38w^D*^g%}q15e5@J_$9n69=&-L2 z{6VPZdp1pVG~3)!3P4@a?)e0oV(`oYbld|h_tSWD_%mdDSnRc`kLS$(ZXP=O^+mT* z-5&f!F60x5=vC+uh#p5d(FUz3QD!|Hz*?BaRbQNn)OsneL@0SMah5thOJc@Y(wg!C zeAhJA6HIAwAPl~=2=`$#QQBgp;vgBC{iN0f9O|RstvrY2wL9m$FfQu_%|F6_J@>ry z{oe{@;t~B)Gm{*(g%~Re5%kTwe|`cj7jmonr(?r85T#vsxIW}bw`4T)C8w!aPmFh) zPjlT_K>DI$PLS)y+Tcb7AM%nT07=f^H!!TElv$Sd`guB!pYq`KmvEooDrbJ5KdW2N zy#M}-FhFuruE;qx4fXy16Da@=Z0ZjLu($sY7r-*NMwnHkq9~w@wlKe{Sua}RtYjq* zneia~YuY$L!O!0JyL!hYt)m!MsAG(O0y&+H1}xy0VY&PP1G~mVyR#z6)Hp^HpqfRXglSNMe|OS@ z51lerD$E}Tsmi{_#zF4zgmtw!$j`_f`5kcQ$hMN zy47YfWSdkDi=1xc2GzyWRGQHr zdl0RVeg1&f?|^Gv^H0LmhGhg`#UQi_q!k+>IarTh)aW9@9&e?*y%V6rP$+$Nw`HS&J9)+QL2#*6ex&B9&j7rqYI z**&UG^H1jMll#g4BH?LN&>4_stMxZx)r*YZT0gfoY#$Oku(jkmvds$`U%$_u-JCTIB^m9mP6W2MHtOh5=wQ zfDfxPmo+o)eCU+^q&D{30I?81>~XSm=64FcvQT4mepY?qR$}ybA$Jjt_6lTuR+Ld#@%`*$Hp^wNn#qFrn3M`lT4pWW^ki(L4W z<))aW{JfE{y}f>URvb@`iY&ndqu%Rg5mh46cwcJaudqM)!9#S~bL7Q<%Zu0LDueGb zFnmOTU%~(QA>bU$E(akg+a$BnWvSwvH($J9O-!@C2?v;&c2k8x0D2dK&YZgO$|z;@ z10s)IowA(fJ`K4s8==wPERX|oNBJ8D)%T>-B8KF{P7B>1PFKuS+uQFa)%gnsMn>!G zRr+t}!S>jw*B%}`H7lQemaiopn_^8^Y1ocA$S*6J0=R*CyDI`{QxX*)+Zm^S3{W5K zB{apdy{{vcrf^5(sQt!lgXY~VE#%4s&PA=(!A-CK`g$g7{elQ`TimBH3yc-uo%|0r zP2WRegzn&s@F5}fw8-K66x12m+am*Ha4#68lPW&|KKzOBYjr!US0v_p`Y<%#2mNJ) z`ZN2W8M)j2&$|JH?QYpt3O-PSg!sVwZSh&TTB@Y{#?hTZ!cLX()V;{XIbx3MdPmj34uyUv50&tnmfAP<4gczsL-dhbUpIeTuz3#_|$sFmphxF++;7H2ir?577+jGi-JHmA`=ZZqefUy|rQx-r+{C6Kj=3@y3K&(O@3;V7|4 z5bK}V_k1lqj7$)dRc-7(4a18aWV`T^Zq*In97?OSh}=sg!uZS5E>B z2i!Vf?!nz=G*?B)_NB8tiBB1tG+ZSu)VjHKFv$bD{{8=8-dtee!3RWU0}X_-v0izZ z^&^fY_Y$qCA7~^k_J<5&NrlSzyv>?VA0-}^r_evjWJ9dl_?zHyJeEG7|86UKv%<P3zke=Ta(&<%y!|?dZ9#_OhM_S5#k!-A!0_X8j1__ejSMmFyGUv>16uKjU zOs8InU$2rqs`}+tk3~ds==FU^PpX{9pI}ks$q`9v!vVy}*oK8wt(*EYKRLeqxTl$x z+3$9@?F1onvt|0$W*_r^JB4_3Qo5{#Np+7)-D8Hpf@5u7vtX-@!V!Tss_Nkb;kNbJ zD_0Sq_N7-NF)B1E|5iCX@9J4&3y)Pd-`KR#eSWBf6&v&ON#KQEjk9GYZ4c=79`-*L zs^2tM%52BDL{OWDU-ruJ#z!THam2v&-WJfC_VvWMj*@Ovt0%%GH>}Q7bOpv_(iBc_ z|1b7Q3qd8<;GnU>Er0qePn%(}PwTW#Yc+XY904lf_ zXE-!f)$Rok)wg)CqrYZg_J(~x-7xzPhJ9gGB7Tc@I#Kkm6}8<=UXQMC;(Kg_JR-iLX{C^sBMIS!IBAYrirB}Yqeof^LXA`qW+%=1+b#P zKm#|FCPGwvgYhiC<8*>I8xMn&ll3YcSB`AXy@8kAvH|;)2*B(C<=#u2!V;5sDR~{4 zORt$?IqSo0n`V*iyZxuA3DL+S@yfZ@RmV_#*T6m}pZb2!2}>K5?ve{XOC7o%yu#s^ zBXX5SpV<~^_-*4?M~haR*vIB`^=gmlq~b#TsKQu8?ii|k3k7|i;v}WWwoLOOFii_U z-dp+(+?ar-5eFRsy)&A7O*z`3g=aaQ-J}0x zHXzeO_?qW3*!)7%Oz5XTJqL6UcLZf@O&BJ_wAP#;Fvli7S+fP!oQ^|#siFu3VD|*w zIuKO=0@$fu>CwY@0R6!YUc{qnLeZ_{j zLwbjaTmZcG*Msw)yPv&6`d(mJ*_PUt?vDOAwzSnYy{$Y2KS^nwxte zQja{ypZUWoQu9-q8u$Y^2L?m0^d%wiA#(>VU$4Hw8cR*3G^>J@pZbYcSFRy0s}2Gb zYZI?7qmnW#sILe=3`yED=6u0Ps^ek*dQ>j@RMJO>_@a&kn6wymVIipkcs7IR83K5u zF3hl$rUV8Bho(Y_5e@U?-wT*XkpA)vez%sqHE%72iXyV$&ZO#nPm$+>4r2&!a;kM^ zuX%g)u83a=eipA;XNw{ipA@)y`W@)u{^H^6+ubG1wl)lmk@DGp@MdA&|2kvJ=Ro7Z zw41bTv0QUh;7pse*YJ6DSjbFU=s2ZAd#!qFw+_>o>GE`~2isN|Q91iwbhwm{C9Wx# z6>2c{l`loF598ftE(B6j_U=MF3Yd@?s`=3T;hDda4%p&sJ&m~{T_OqRS&57gy$>~G>)EE|~YkB?gw}bWb z;35Ii&+Az9$FP2{h4E+SQz`7-6w}-o z3iFv3gbX!5VS;yVIIP)t{(@&LNocs8(5*Mc?hEaqZ)sb-2QY#`!5e)y=10ENzuDOC zN_k`!9wITTE~7=H^*XGw-8Y$e{*<;XO@N@YSx=6pbD@O?axLIVe^G~;KPS->BK9Gr zp>_8#+M#Ve0q#zi`^gdpGRn||p}0V^Uw^)GyqEU>;U2W-z1=N07T>f@HFg_uVnhm~ z4@75D1fmyjAB1>!F_&Ku=#F9&Eu_e?uM)h>PpZ=P@NoXBRX}vK7j!)S!55&W0EKZ*N4*MIk?_qT7OR+$JUSDVHar@HcV{#Joa{nY#>t_r0?YjHf zk(=f-57#c=BIY=0Cb82^MIKISq~V;tb+F7QsJNI4fJ+O?oLo?vOD9?_UU!l2OnlF+Y*inDaRc`INy^eW&LVu zAe)i7tDJ1PLPz@Galz<&rsOVy9c#+g7%uNR?TKG#%tgKFmMAurs%B_8emiAH{ zqAg&(`f-Wht#{hLbMxCxFM7h#Q#T51peW$!Q26b(Yu2E{PWN}ou3Qz{5Q^0|eX>6O z;wZD5N(1LWNS0NO9Fdh3i>@$Av|g)35zxl)4FrtuLE*NGztAipv17GRFxxij7$S<6;FO3 z*wJOL%3y^yFi`9NWUi;b(DeRs*y}zNiN%0>eMRqa5(vT2^{3>nj zTSGeOL%KX~_uyZduevba+*(prPGP3^L4_*vNV4D_)XEs63kWZg0_oE_Jm{PLYS@%p z2J;#Hii=E(pU4mo@Ww$7+jx#slYJPU^tlAzV98#u2xPs43hf4af&E?jrX-P@05P?T z`}63XCNe<%!H}k@6I1(u$&ZlNDYWOKFMaDs=AV47RSq3fr&Qv_7YZVO-Wk05ECHrI zH^V~dyZB(q{;p|v^rie!NR=~k_q*cRpi?@0YQqJ$Eja)Un5^fRD3))iQDIn?cS-iQ zScA6Jp0Zr#v>D*p8vtZ|XXSh16fjTw&5YeTj1ZLu>zDnk6=8jf%_b7?$)CXNqn6O&zDORp0q1HC%38b zg$KcZzg96z!m%p07tSHQ_K`%KOqUg`_9vouG0}WrPeO-=Lwp(;W+x~`0wL%RlTfQWhX>bEQ%EC&_s1iy(5baF|jX6Q@A{#tOUbfmr83jF^=2uz$~ zmUqXmgJno@n+StJD-^HKe!wB7wbVU(Td#GQ!S7k5a!BsOy_S=6ZJ#H!Ve${G3i5dQ z6uHe%Z+i=gW{FkLp%D9WKPdJ1%i$35sA1fUMEUdsvZQZ*v?CFYzpL@ z6a>jlWdfyyLRC10D`>Mz(*H8cLVrK@w-QJdt)Pg2(he%h55gsU4YT+WO2*O;fqX_K zcv;ojirkfNf6Pl_Zs@adE25V!hWdyZ^KygnC6A0-b2;2hwkzF#M_2QMnqw0Lm>fhg7ne*Hv@`XJHSF55!DYib4>RUZR zBC%?1Osw{zA2}~yhrYRoA3>qcHFtpq1unk9S^U+3X?}86JVBktX4lVOrVI4b^&0Dz*H~=+t6vn8v1!CcCB-1*bK$(^vrdWyhn~-`V%+ml6AuO z>Rvz@E)Q{V2mO-sy0dG?HA8!GXxA^9S=ON>qe9mN6X~-{h=}{5V&Pc~ou^RT&U&^$ zwQZRyzI$23^$+nw;)#<@Qsfn~J6>`>akd1k)=;wK6Ql!cZ%|G*B; z=49L|iUnVM#9+bUXHS>BTsU=RNY&I@z!;85Gh;IR4fWi=`U3vq&8O0O3UufM^~R70 zvu|CyqPNz@<~Ao(I7B|4WZ(W{ebnimB8%Azp^@$guKR69@4sZ=|Ee*~3_C94M$J$m zH9mCv{80G_A4^o*x7tKu75BBF z&O_?ek7roLn?cWFIH3@kd@>L@vpKgBU=|q+m%4BO@B-D;u(dyt*_G#p4d<8%C62AP~F2YqD@ z@tZF^vBnnJf+DkT#G1I<-oGFq+gVjxwMGq`J=1MYlwZJoMI1m+MeoZCik1ef?~o+g z+T_!lxqzv3?YWbfYnkkJuUkvc7~l+x`ouKn_%`2q52l2uHG(J2Rj^3VR__*@J|7;~ zJVtFmo)PwwK@@v&S0Vz-%EUkXd(kPvC44hwnUC8)cT-|I2-r*XytCM!56mL|1&6m+^6#nc@jaaY;3)prJ@W7`d$E zo5pW)7x{ds_Q2Nny+xw#X8YJ9{z=OOu#jkiHg~XY2i+=YQl;zYxLka0q1Q*yNtw!> zo{9IR@DD~D-ky{vkBu%8HvSD6SrBbYE^bTyXBG~4U`{Fl=-K-#Hh5UD!Ts;mVFmou zEC~XAQ#J+F&KSf8>w53kW$-Ch#~stq9?c#5Li6u7(tfWd@cwu)Yp%oQTn7!{c^ycv z79d$vz@l;|zz>7P$C%~Fn^Nhs`Bzn~JEKDMTCZ$FO;!);#-GR(Q3by8 z(J11}lJ;`t&7?k6{1+tkiV!r<%NF7^p8aPgj8KwJtoMz0KBb9)O27bXo{S(J6}ram z7lTTJuKVWhE2LYJ$Klar^D4hNO9T8Hf;c6siob{0uy8nkw#2WP))OO-jaBahl=5|y z;c%<9U5xxS5i!vf7Ea8Sr13AcpwpLPZ$qHa;{O2AR>kK zS7`E$5{qoopW>vL?W4VRODdgxkftvk2IybzS_E?Du7vBeJY-d6yp=;!_&JJbj#jgc z{)|PUgFw2T-ZKoBq@W?~LX~VTs(agVN|xeXc7FI$lVWu&ftIuw9wy|Se)r=x39R!ZA6ttuy5^A;!nR~UrL7u@#Xvg&TP#Wu&>;Z% zgb~Ko=X~pTPd6Eun38a9i#K$`u1_a?L|9GonExRrmk=~`l6rFpha9Y4z8-!LpF0+K zGu~{hJ}eh1NHG-c$vV?gh`q3{Z=6Gm&zT@F_=4(0%VD;L8Kr@~pTyFHvcxV>saESE z;DG)S9lC*h!2p&Y#%=onw7N5Fo=zAnkvt5$=Yk`?#(6cHL?XGI%(L#UcJrrlb2NlG z5_Fg>?OtrwZ-qIr)1g+|(4O4QYyLIezwFDspan zpA4g(y@lLzmD7A*7bpAB=Yp9eD<0BC@40F*w3Vpn^3)7?pbhEXeP2phBEe7nD$jo} z&)%09B7RV{WN7nqx=ZVK1kazsD1tFo99?I|iA)+ox!?C5N8Hl5@<8Y-OU((eER_}M z>DG}|dWLBqY&t*|O@EU7i=wqaLw^}f72E``CCX%7*^4>WcjIJdE#f#*#^(xCp~BC3 z7>ZKxC#Ru{QV84x=`WMf)_fFS2Pnh^Lp3aqd}zIiu&Ft4kAjGrslM10PK6 zrSsdUc-AR4M;VMsR!X%IIij`hnA%vk$f%k;Ff-*f+x`6?F2GwxGy3l@$2(Q*LVrH} zv2gFNY)1;apkUl_5Si`?JH6F5cqtCRM=tcq0I@5$mH;n-!WDsQciXzI=jhTEOU=?} znm)EU!^vZX|F;(ceWXXl;e6b@A}4$3#}fv?ih3j;+!(sC)dduw}DB#@V$BGiBDxOQ8J@qV&X@sgPyD=7|$U8ZT z$3>*2>d8opSJ{!VTAN@56CpudMiL>gqJm9hkZb*l+bs*10{xpEuwr#xxmH7#9;H*YPZ;E{>|aq#+VOHFLJLu2Ey#6zj&pX$SG| z#>C&%exFk!Y(8@s8`WE)zxP3MS$S&0qb#YezefGc1>ZL9H{}xa7u3J%7UBeg`ps~4 zi)i6tBu{p*BbSP3UBJ?OrTWB}4e zfBe!;=b_Rr1Vh2fee(y$hu=Hm3ZYNd3;;2(kU&Kr&4)B`POB(Q?NqLwK%G<5k%(d* z6tMj5->2IT{dEtA-FKGVBPvW@pDuz{DBWXacOByv5omKf+X}Dcono=Af_l=FeH>2f zdhx`>kBlz-+->)8AwX29opq^c9=^v?5QFay4x(RqFf|8BP&yz4+UkhBl@>5hEp#C(L15 z5&QVdV|Q zE5}QE$oPWz)WE)BT5g@M)WyYF!k3hTXuklAZU8XGM1|DZhh#dzIoI6;63#y-SH~8_ zhsh@hbdQmlb3iUmee{fTZ29{Wd~3pC7kbUc(@BpFmK$|@(@jCuqU=p!A-rkxf$o?! z403_H1Zkz7{|@oVP*%QgBz0hl$F^s{u_?ipeYvf?`JH*P2oBsBWZyC*t(M__WE80= z2-lR@&+fPP!a!XTRwf?YTy${+w&EX7_$Wo}ZgvK(!0l)z-fU;&YdOFDv2NeFoLe4y z;D62T_eri}UvYZHyhHUU3Kr__$Es}be{7e%esy?Bk$xBg(SB%u!SX~{>uIZQST8^K zFD}9s1?0z#4ZTl^<40T~sd@n(ejBQ!_TsovVW1t-SQd@dR@8QK5zWo>2jwZ_K`xz# zXRm>_ryEo&?9nU2a)D}Czr`n=`GDH>iM+`G;4&6~{BwH02YE5Rtw!_f0!=9lH7^?x zY9C-)n_-mWxEd@cc?X7fGW=r)LDeMI_x(1`uc^~}O5T!|-ZsD|;RQvzCK&GNc=c^c zQ(i5clhv}HbLr3R&jORYHn{8cv}_Dq7a5-4Ss5!EaIIn)91`2ZQGSGKwOg8mMdLo$Z&xp%xn2)ZLBEko>nSZ7qO=}S|S zY(6seh}JGWa({yChuIrVyxj+@dO2!y;iXjM+0~@=w>hdd(u^P4JWU=jdbsF3xc8ov z_@JD&L9FX4JoPqv|$7XWnuuutP(pED)zsJ{>KJyRzN zIC&VXtc*8Cz7mO4J&K8bF69$L0oas`@JWcjt@3Q^j8s>(56poLKmn8XnnJ_Tr)zZ- z06p@X31t-(P;yr-0W(j3}3Y zBN7@HB?^?6RQR-yVC5cF2}xo)UEI4yD)?F4v_Fsn-v_Z2{x^%^VZtQiP#f<(bc-zO)J!~+8{9Bl%rWewAQn75wKQiT2k+QeABszV_sMv9)O}g)ebtpmmG2|TC zu_SANO!7GvT-)^~-yWouYL1d)B!(Uc<0qprp@wN_;`-%%Hdl8T&l`b(nIj_gKI>C` z>;oZ|C#*PD{1LxCs6O^u-M_%xC_M=*gy!bjb;jkr{-mqf-cre^PFj3}LEhcKqeiT- zE=HMM6B?De2Ad8HT+xjvtD@%D_e|A`K-kLO6?1+Na2v7zlEgpXspMKkXe7;W$Rwb%t(WLL!)RYQ$a&C zw7nE*Y@9tpUYEFe(A9ixCo!+~szJu*2C8aYCn`M3lCrk-wx398kpG+?$6E6AfU#9$ z661mrHrU^7IzgQ1@tEtV!0GZB0~ZSC)-iEW0pK2^2I5Hl8w;~_e@>7Lnp z1l1r$y-=h*u}`Ja#h92#Mv8K>q`>RTBZbUDSL!}|Sn56MrshfMXM$^CGaDY(Im`HM zR6r^gf`(3@NUrZ|#nO$CWF?<=xb~)ZYAi-zTB!V5-9{n2!oEl~QfK{-|7i zlYm^H4DNf1HS6Eb)b;bS*{v#CvlaAo;4;E8&kc7Rv-ORGQKApy;)Lb^g^J%e8juhZyI`R6TW9dXette#fwdw2 zhhGeJ<^pg1;C!F$jySK)8FA&&5*;z!&*qHiFp^|b*%!wGHmBTFdhou^iJdZgUai9l zmh*c{P7yi0@$VCBXHDc{$@^k{VGJFhYD8syFft`CQ03Wt88Xc(b4tyVT zc6^>JjMsF)|K3B9BUZ}forYC5jSYY|W305(PnANa$t8FHbIWS3nEEVi$T!W)+nZ*| z@>nktPdV$53}T|Y!jMHQeO;a*;eD@pB&a-lot^qtMzcHx&XChhR~yYwe`WIHTFsoI5d zYov*p?Z>(}`z5ri6UdxwhhTJv*j<6GSMT0vN}0X44#^h)yEUB$$>YZ6T@~5X&N68U zC{D5+%|e2u!<60yIWO#p=~)Ym+gY7c14D1e7qb$4%*vaauJKm=lYZ3UFZA7-KEp?? zey}mKb$JH?Zsee8_Rkm@dHHZfJ)|HB2Ag`!cG@p$sFYROmuFgugM`2%iEnOO-3P+( z3B_|Q>J1bte`2Z~*(;BLsnv zeL9t)Vxt|&s3n}HEmBh(r`clsRmJ`m9XJU&?WGsO14WtLoZRw*LVSpplB*-G4CKB+ z`4aP>_phF7A6i?583aa^fP0(Nx&CLjR?fYlWS0 zxd$St*-0M!0o5MHpVajOJD*7)5dHN+vwwR0NlrwGCD8Z}C@CB!RlW@_aK66V{CIcY z&X;2AKbw3DnGVTc z!8eW7@)s&h*0NpO$MF*%tXCvQ{boAwkLVDA?stoEbFZ`^Zs0)6rWZL zCOIi1x~})gHZ|n*(Kk#n5YJKj?N7MS2gH&)vXstYds zfN>IgIvjj2(3OIND}Y*ri4HHMgteL&oBr@_IceVW+9&Y-FN6Xp<6~5c+pe#+F{m~s z$ICC3TT*G$$L?T+t%LIp>#{@*PVlAn(S{0jRSNW;#?WUCLJ!|XWr#=}UX!=% z1zm&W)=R`z4CA~^?y*2ehms69qjbe^b^qv)2%AwR)-#d$GuNFVYEQWi%`;o&&O^Sch$6?|2TqkFzkK?68K zCjzDsPm`KmODbrjw2r(&`P-4rGkIDKWX0;gU$O1Xw;65t!eRWO%F{4=Rm#!IQrPe= z5-PRvMj@0QvA%|q7;X(5@xd11;1D6MiS(9fF5T|d+NQLRr*>A@D=p1mx)-T)^&|kL z^J$muXls015TdA`^O_=)Gus+kUKNL$2NyH#IZ$TeD3%X4;<_6i?#DErT6@?B=|wKgz7Qpm2G(Raj9;vLxP1K!f%3E3v_0*m_mH?N8s zD>a^}^+af|Mfoc1rS~&KNoVL>wY|!yj2BK*h78{;%(gs8pgV(e=7wmq?Exvxz(Fh|} z&vc)y%X=Wzk1cx6Scwv=VSSk2;Wn-SuE_-b`qP)vxWBn(WX#(yTAx5? z=rRKma#;%rULIIi9yJj6vL2S$y_ZqqjR-kUcAcUTr_FH)LU*`VoQ(hYWcVtk!RMg9 zqT}taqNq69CoP130d)DHB>53v!*Omb8U#q5$6>?Ex>?; zz{Ex39}*b-sPv7$ID+cAhp``fUIO2GN(&Bpc3qFnH9U-y%IZ{djQ4yT7-Do@YJk4c?Je!F$*lHyRVW77{tzVSuG_+a$F+Q0^m-^ju2q|*|Nck( z@;-FP5gb0FSh6tt{Q*pV6)RE%aS;W`KWfy%#10vorgQvgqO9ii%h(E7?NKkskoF(X zY<5>d2fs_lvv?1c1D>A1dyB0tRcHg7a*ApdM;U7to5nikogFIB*ywlgZvso6%`f}y%wC9?t3>0=i zc>?`Nd@W&e=q6)fR`6r(M-HyNCdEBolsNZo@jG4mT5PFVLlde7$BIYORZD%|-0Jh- zVc)s&1cu&Os6T249Ma_4Az-1ce7X>!z0sht_|W;iQ`Q#uC-)#88ev zqYLN6Msi3r*sV=MvO>|`T6Z^;YZFtqJZJ8kd`g6+o-{6gj3=_Gc)e!htSkV(g%||7 z4>5eOWg87cn3$x~I-|-K&UzQE00%5D;kl;L`Ovcf9`bh_RyoT;Z9J&kD1rUi=i=ek zsjud)wcvVEj{B6KoPslsOV?!_X!JrCKK4BQZaCmDbaezYNYf0nQl1M?rk)9KXD%-! znxoC=j)qdWzt-c!EQg_)z&96vkV2A6Uc`jf|5Y@=uEfC6h!0A1!6>LNaVsNKN^eh@ z-9n4pN`)XvXmAtfp?Wy=uNhuKsqUz(z1s)a4uTF-JlTAPD>3!E=#dm{#45icM{D5o zgKLbT#$zhEbX9bCylJd4$6{AhuzmH+hMYQMM3k1B=|;Gum%a^2*9a_%wh&Ey zL-G;!+svj~O_zR8!Q0-+lD=$-Tm4C$`|8Eb__an}H$Jxb^TZ1aF5lEqtPQ!v2+q=DT;1D49&?V4IvQZq$6(;O?b`5ODBtQj zoNRo2BPk^f$O2hk$C)!hxLZb)QhU?9cJvT>%1ot9Jt`!`W~zWu@S;*6=!WH?5rVXY z_(z^VjKr%7iWH`w3yi$%C3Op@)s*73ry9x>Fw2*8j^DDc(f{!RD1thUy!Mv2YxXq2 z);$`6HW3slgBY>*)Nx?IX%JKxXwZ%-L|hxR4|wOM7+ezYs$TlFKZS^+z2AHMSKbHY z&_bmYdLyGhPe=MVa>kVXf3C@_gN1->o~1z=)FGQ!I7fhdY>gbP+)D4N-DwUk6?N32 zFA&|ang&52mA1W$3JWX?s5t1Y_i$L?5fO)}qnop?SD$J7yFF4@vSzWRe_~F_P9Bt> zt68}GtMf~q{Bmmngw(l(Kcjj7Z_G(EccznGk!WaoZrnQF`C*h zcTafLMev%xC%I~fbivm-%{u2VasN9p@0K9<8ngR%)+R@fPu-O6+ZT*k+Ag-(z)HdH z_6@8Q%s)0TE+c?X?i)bh#jZ5NfA;l4DTgLWrMI`s-W2BoXmq)FhiR&v%qO?Xn9 zFabKO%^Ljq?>ZYbd6-6se~`>H@ZKT5_^SMVGaxQNdxO&n3wd<-c%n{waiT{`4Rm`;1A)cJB9r^%@pD1nO_k?5&Jk}rbh_!UNo$*mvBjp9x@r)L% zgm+#(?%}3~9XMZcE z61GlX@4CdhmxdP>5;c_a;37?h^Nknff&667PmP7ELeb$ke@2+-$Iv{#5V>=Wy}72T zc zTPEswu#BJJ&d@#G2tfkHxe)}OKnyp6NUBQoXx>`Ldig7_+@&$$BqZpejY03p3=77a z`NH@4qmKkdnqR4A*>7f(=urpw2SCy1TF!1F$;XKeyQXZEtL7M(}_o`i3vU@k=bg3hR zv#fZVf-vpAGb5nL82tM3XX=$6KuB7i@vo*+<#`2!LT`o|~ zuH=hb2s@NC&#tC=J6%}`3vU{?Z4L3<*o;+WY^kDTZ}C3*0(s?%?YSYoQ4b-}q(~!$ zAvHVDcO|ZM@-@bupDlhP1k^tg%DvC+_M6Bf1~F(1j5^F#BT8*G0(+)vClYw=AC7!WPsz>Po1khhb^tu}i~6e} z8W>MWXcrsb8v44NH__PonA)akulJ^%dg6K=1>Z!Y(jz)m63I*=pVA05nPfPzVu%7+vHi7cF*ym>D2=p%?JNd?2 zW_L9a)rf6M;RZC9eTjv?tRG%Hx{m;2raqmG`&Lkpqf+tfTD3*j?QwVX7e`CVwYbnURd<8BZYA6=~UT&PRwhBPM9sd~ec2f!d; zjxw___;Cc3`wYJ9T_jQ`a{pwN4?0%op(LT*e4jySQOZR1tIZ~J{?QoiuTVXg{a%d0S@+Loa?Odc*WCQM8uyzij!mczOgr17WR)*yF zBh~CNE=Gx`k4u~W4_9v)73CN84G-NV9Rm!hG$J6~AqWUcNQ2TPT?0d>q;yDIbV$Qc ziULxiNSCxQG($g!|9ju-dDi;@<%5gxI@j5I|7y1~8pr!3zxz5QZ(~{8A;}VicU|nC zTXn~GK5(%Nrz6Dx&fpmDHOimav>5R!9vbAt{djF)=?!284=ZgPhjX;49Gik5ag?&*XMgh*8cQo^!#PP>v<;65nq)m1z zqZH&+YoZgPi24lg23U_4&eZNcmh|#*?WBZyM)HZ7$Mjarg$unvsF?Jw@+uqrsky{J3<& zKhc`Du<%tfLz^Sx$=An}J5(QU{~15)g5pAu!H=s=?L8-2p3Zy{zWm`2fGRc6$QgRR z{SCwbW8Z3_!D4K{-#Su+pm6v7L#ml}3!9gi`V13aJ^3d^I(?pz-$?2`dZJQRDmODA zvhXP1JX?^DHEkyMZZP~_rT@odvCH@TU73|Vw1-`%l>cow0`hB6Kr28E?*iyG5sP5S z*;O2);gMT0_sxTC=wzV7AK*H;orW*)_Ad^|<+6|u&)p;M!lA$TAXGY^)&g^i9Bn zDC7Atc?0Cg*qiY|RQ+g{gyjN_x62wU>F+Ec<`pn~G`D2#0^} zj`iaW=jfLhIzzhyI64>O`n;m8NUT7uJ<_EK(GrDWSg=pKKy~S4J+UU1mH9nBAwNP=rL=3f+-u&Cj_4h=7zT zX#4aA#eeuO)*h#e*EGf+7K1e}^-EqreiNcbI9A6VX_4Bn)KF}{RH9ryyk0OQrTQ8A zO8_KIL2eVjT{!wb8@rw=R%k|E@i+Z``kP5)C(EsD=dh!G%z4nK9FWZ( z<}2<8aTwPXiNz{iHn($?W>CE2ay(lgOWarK>qcuWbP|=#cZV_%I6>1x6^g?ZOt0p8zRC4(%oi^8Gm0~^?Li<$}1OM#`Y1)5>Cumb?XHflK`I|gX{c|o3 z8j1`i4MpJh3C6O+cs)E_OM1_Sslx5Tbnu5nc@pCF@pC!+mZnNB?RKaWaw;(bGfeem zupt9IunRqbGZPisVK4jEM6@8LcF)v#Uck;X3ceEn>5Bas@s;_dZhiW5!z6CEepB!b z&>JY2sK-e&Y z-E&7>WD$|W*ZQV2P2=hplUW~HYu?Uuk8WwREz4i~s`1>rQleasl$-QSc5(7s7_R_e zu0~rX)syV9(2JRToFuazGp&W$!Ui&;yY6}V6RK(geFlI;n;`_QJor5pI2nzk^`2~E z^y!Zsv)xlMlG>l|U_QelH|PZ0-2CB(v^{pS6;}2$mxRV=?Im9{`qjuTKkpFK9)n-0 z5z_*9e$aS>cn>-c3Uo8(B~DW2SQx_a;kt~0@?e3pj(+)lIhsm+-R)5(}2ao zm9>22mndGt6w6G)2AgD~9!ulLtg`i_ zU2lZ#a|L8XCm>|uemjykfF0xE<#9FQ%5%&x(tGNn^Mxnb-x}zh+g(d4&R&?fdqE#> zC9!ohlJ_Au&oC#oxmUsZJ5qX`%m=+APww;zO41Fd{p=P!^YYRR0R$HW%;FZm#!sri z;@r<%pUNG!mft!cx2mN!->iy}PHXBf2G)#z&vPkqsjp~o$w5+gi2KiZTQqEVBVggR-+VSL9od0DX!ceufs_^6LVF^Gp#Xk?Fb3sv9+%8h?}J* z;(TFl`&%|H?;7*;G0eK@-HGPdRMzoDm&pAv0u8|F_}DI`dGO6JC{=H|PH# zjDzUgHB#WYmcehCqNlO`N9kfo!*{ zW>p%#tp)oPU1@W~cpGAZ9WN@C+p`s%Z+$ZKiH-IEZ-7E$9RwN9??okx1+U@{C@ywbdXfHM+r9U&>U5&! znH^B+BZT7R;c_z<+R~D(q(Sr5N(-!RL8=S#3uDI{QUcT=P4d#1`$`&@aM61{5A@3O zn0w6gN2l?({_Q|Iqn~u@;3s=^&jX|93zY?F@(=?!2GOc0R+RP?drSbfb1HY@ zCIExaYB*{PYXQjS|G!T_Pfq`m(K4hUS?#^onU|!ljz! ze;u0`(-r9E?A0f|2gr3}3_wgXZV+iHeCrqkAviIlZr)GUoa>#FE19|$#C~dI*EeSf z|I|1s7tfU|;Z2N>`S3Bcj52n$l<`Gb9{~>NnuvffF%$)cyMyl2jZlr43Yx-U97D%1|}_V!?Fk*Xcf~p_)9rGQ03W&Z`luIpSqGCcsH_tyP_w zE{yyrL4my)afka!@|SxPSGvB6ZxB=rPBWiWiac~dGz{;j{)0L23KGuwYw!6n-@mXb zouoSa6?&&XAEg|iPs!yBoojbtodDIWFSd!j+k@Wm4R+rTC&d6xt09A7ZEiYP)#Bvk z>_K#)XmBjH6jnNZo@3RyXx*rBsVCPRB7#Bf-7oR&v|bOxK!hG`U;d1p-8D|^m$T6r zY^&Dc7KhbIVTofL$hC|K;gy`7H=BJbsPA7G;oKQ=@*gwE8N*A5bm;8C-Di%*4gznK|G}iY@@<$?a4I?*sy_g zUl>Gi6HI>Jj_Xvfz8gI-dXla9V(qJ`trq{e&SF*dtDDY^gn;8M@0rh6lP7=wwW+pT zc+gq=CxZnF{cEt@K=3jIH~|@<^tVHm_nDVQrVi+nHY3E6I_F$C?bvltir9)oHMO5= z5)++ zjfs)-wfN6S9iABXI!5`pniT^ej|TM{Pp1!SNLgZ(j!TAFUDjr|~D)a}8Or zV7Z0MarRkOri=S|vd*ps9J*I*0&jnP6T^iOsb^4(eQ$%F-%OOaYB!|HGUQNb9?WuH zV;A+|Srec(gBRRHm~-iX*`0X*;rWO<)Tb8eI>Gu@ z&A*CNUrT$J$v4 z$gslU3rq$gg%_?f>+UT~eJmaNIIMov7Dd)xUurSby-b%Dc*vV<^m2{Hnn;_S5)`a; zvF6P#xFD>O>PeFpDy*{aD=JJ1_2#0!6jZA^(hv5^?n@XNcCiVZaxl~1h#=RYATZE| zUr7~5;hrOGEn+}fU5L5AgKUnvF1@EGIH}-J4Sl&9GW0l#Z2{Xn{i70jq$}|2WTiJ_vYV_b>b000Z9oS$?Z*bD=YhL;EBH)1y1I5PkSQnNFv{7Y`Hb_t$kiM-Jh9A6Q&X` zj`-cV#n}${(G(msSlx(G3AAmubSXvnsUBp(m+X?p54l)-KW8DyiRe82=WPD~FXUE* zeMqXWCVz>|Xyw^RQL=vBI^|ka9vY;k2rr5|7Q_-SL}B-ou(OlPnKI(--veI9XMpX1 z1`%a2&kS^I31HNvpJ}PzV~1z8G7zea)NDLmqMKjXqp7QsDjLeftpOc)!?*{rfixkj z5J1hr7Ib-MABF~lJ3q{y09)b)E8ms1-C>wtjmC#>3ts+rmn*qC5A>{=Vjm>L0vtPX ze2;U@c~zc24DENN3$|}P<3h@=yK_vt%o?Rs^!T!sH4soza%1c@NoEK71j3K1i|(5% zv`13!1cF{CLa}F!vL16U+l+Tf!rL|n(_z(A{#EYEE%oxekJhv&91GFyMsGT4VXp*V zEXqVFIu9Ebhc*Ipnj?KYeVZW${*E2dQj6u!+V7*1>JOeh2eR1ux+l4Z`x}g>2PU|a zS=)RR2+ySF6o)m}Ph`#JGNBLuo0@eSto>aT7jw`wjVGg?m; z&CKU@31T?N5r?^v4cXCWjOMVwi)#>k=r-d-tQ_hgW`&8CnzQc?VC(q$t z2ND(15_j#V<6exgiCw5jKindN3&s?fEalyFf^4~)8Np@2v61BwCES+JawPN-NRmKO z1j32+TH+=5`D;#T$pIa5=1{c44Di;%IU!|}vS@a_0goav4MU>xBVc7UT=_z9mD5Dx zO>j8(VHK&YI;DU2m}A*+UavOWTp=sW#*Oc!bW+0s zBUr0;S+BBo?AR{JeN@07aWOG^b`#!L%=UrArQn`1-#LR2d=4MdExsnHiD~~Z)7vrZ z>_hgC`M}S>suPJ021SuEzY6#EiSHbDfdhaOgV4*>vG}t7)LS>7Tb9=A4(n0N4yj~~ z20$`G0A^qSdT$M&CbTKm$)NjwZ8jP1Xwi3{k3!RH=?$Sqf$j1?uj!|eU}dO??W>y% z$zO{7&p6GNV+Faa<<+&8g>=6Y2D& zEi9-PI>DQ7^K0gK4lR+x)i#uP4?yf~kWS?71PZLyO}&b0q>Ce< zlk&VsA7Acws}s4P=Wd=i>D3O6LiZ?r{Q8Wn>{>d_-;O-@Le$zb?7hp?F7ddAz~VSQ~CsoON#Cmm=P7f)Fmcj6_`me;1 zpAlE1P*^9V)cmb5)AP@Ox$Cnk9_V7LuVX;Km)#&?jIKnlPKrLXk!N<$eCPCL>dQ1D zcXS03AG2!E zukP<6ZvqwYQk1BdDUX$_$C|9pXc;)DQ_ z#n!ED6m(8W%yNBI$mp5R$+q=KSw>dp+|=cvbLqbhL*OSY10xyMt-S0Oa{Mnw;d#sm z7QB`RUX=f2RrdrI=y})XE@<8}Pib#KgF48~6Ez6DbH{-0!?^AU%81q{F7wW?ds#=Q zteNg8ll?YX*TQd5S^0w`@ea!48N%PYnVzittK!k1GaPlU{z(Dk+^TAT+`*o@U8zS1 zd?=eFyX#+XsK(C?nW!#Yg@ZZ{C4c?$C})2C^Kh0mCmS*|W}iUEJ^JU|l`v$hM`>eOWfef&-#rk3%PVUM+bm{7yUPaz=h% zoH~7vk$kPkmv?v*FadtkRVPkXK9 z>Q*9)hd+54g%x^n+M<4TC^WMS#cjr?9o+xU<5(0RP|}J~q7Upe2vcvh7{RW%?~H-$ zMa_hc;a<2(#u7_c{4eGji3|vmPxQaI6%A=Av>$Ulv-aVv6Z%!Js z@U{cmcxsuN&E+?v#s@7e1|FR@g}jzEo8vflmv@h6*i9iN9TtJ)4mK)J=%;ZvJyynI z(*^?Ck!vK7C0*#O^a4kF((3^(cO4(frmW)HX(8kN$qbxKR^7WGbW!0zM{qv+uXV>p z7Uf(1MtyOsya1+W(zx~f9tN4ODWD(i#9f3Xi>WSw@ z`+iJp)y|034Ax#5P$2@TCqM$ioI8M*?LVLzinFKVc(~atnWW9h@Znun^;f&F$VLs# z5cC=M;O3}g$LW6^F|8lc7w6P-Ith8i zU5qhEqM&}^4Vhw$4mDwBV{q{V#)H`}Z7*^^{p>2`xfzj7r*d;~EJ#CTD`8{7D!C!| z3h70->rV(j_qG0!eH9}}UY+Mu_13fI&#PeZ;`bnqa`E;CszG_mp3_@bnaQ{7&*~|^ z@BpH-yVW}iM-wa&eOXWM1}KKU_4ECH^@P5z=yOdmkzy6_5dP!JfS+l5j(R+NR;=-{ zO%dK5?4$?euYyto)Q0MS0yq({@y<$n*y?YJX1BWVZ4ywn73F2HFBf(%)O8s6x;C+J zup)%6mXIJ2I?LX;NY=Ia8$P3AS$7-pgj+6GFyKZn-On6HZb9eC{(_6Oq?LlFRvMGC z*XGAFy8RKmU7`{@BG)06gmeB>!fFqqU&TbN+NJ6BS*`g*`lLxOPR;xDiB7W6zx!A$ z=6owRraPGirV0iFRzn4C0cn8(Q6e(8z#+;817u?-Hto%Wg&M*{`A?Y_uO7L`=8yjU z`Ka=3XUt&9CFK{UH$*tV;a|L?CmPf^rXKw}5J*mNZ|!Jcf0SVe(zh)31pcy=J-70i zZ@!=k=|L6cd1Yf=tPgKRLr`L5-P##G3J9+48Y52?HM5Gxr zd5X1TY^pqPTJhr-8i-YabK?h=1=taXJO7`m9;htAk9nP((de!YbAg8+4q3?j(av4q zEAAa|{B+N=^*Eoki121B)4O4p+MN*!2ypR`1!@4fgU$^iCl78}?5P3$#Su@Jph0`{ z1wtS50*{=T_w5^%7hKnLi@VikK?P*Zl6RKzoZ^4DD6V-c-snDsd{er-30!?~1Xc6% zN-GHc3@)kze$cuo`?fbCxuSiX!x7V7;g=PX#3WU<1T5mN}>d!L$ zOu0@)i3`su5aUlz1TFYaoB#$yAb$a<6F7YZw_S_V9PFCFMKbzxk>yxJIW{TMBtM8sV$_u@XVv>wt<-iR3;+`? z*qbwA(203(O;vro7KJ&s@h@3dplA8T7&(oPZ?x=5srMF!U_vnVxQ4GnTp9fx5<=gm z=E@TMkG2@8Bx;LS^fGNv#2sh61Y;I@TLk5dkAx~Z(?}eCtB&4f+CET$CK!Ya-Ir;H z?l&wzSPEAU1NM%@VeijLInwZl6mnyYDee!rb!XNhutKVSQS-Tlht@BIefCaB?xL&V zaiPRWtrWNP@Aia@x;HA=g`*FAF1)Tilg$F3?q_MG<;HKoJ?)CD5 zaOIya>Jzb#%M#xyhdi5s8nvKYAR7#Xyl#}cM$dmR{kX_b`u#QDM8F?!zXJzN$P7;+ zgo;86R=UXux(w-FmgDb1y*bLT$@)x{c4{@KRBC&b;4PGBTP;fFn6Z^z8%}_8i^rfM z75nGZ1^xb43qY_>yXAi941LS1|O{NW**d!*!+#or`OFR9Bgx3Z&ft!UGvm>8zb(kb;>=6js-%ZL*6=h(+Yni{Ei? zE}kvsz5nu6fo#1K6_B{22aC}mX9;1IRk++`JPW#Zn7#Y$;~Pn(=U6mvjxbBJC=Bv$ zaLtKt9OMi2)DqgwS0+^ye5iSR2D#`tSX1SY>QQt&ht4LYC2(8fccN$l!IqC~AdU{Wz#=y$nA$NQl_Jd9F-xs*w+Rqzr zSqnqW(-5>w9yPtc_|VOk8+nj0+5N$s?voY`ktz*JDW03MmGqm(I zmhWf;Sw0XP+wKV@UimxdfG|({J^IB%CNvOS1;Phk%^bhsx9{NjVMhLsIYUI*U1na3 zcGprnf`n4G!Ey_kh9T>l>>>n_RVrD*-IEA+{sp;AU$PlCK2ALH+?R8qlWJzt`Mb~; zl{>CFm~EO6Z{Z>}nt`()go&>9U_vt`kkT$9Kg@@s>Q(zq@Hz=Y1QjkF^QPZxWL-R1 z&Ly)|ZF&2m?HT?uWgi|ZI624*+*-V&-5V6S_r8<5aLGZ4NjKlG`&n?}dy5;& ziZ)jat=Kh7 z5e#zGGCG$_cI8jqJ$PYW_*uzAq9^Y2IMihnXX!KX-%REl?Ce!Gj=Ev@M2x0QYA*Dk zCz;QtYV7aLsrq$a3`k;?L~G?MI|B1r_^(}}_EXy!e$Aqi2Yrn=^LP$1J2{Ffc%3<< z@Kac$3x@1q=m;j-f}46}sn7aSvIUz=*{nB}ofG8-vvjhouD2nncpx% z?00`bc(D;{#4Ne(z==<*+0%ylD~Z?zF={4Yj~=q%hS$lWAq!0m z?~f`QM>YbX{U@!>KaQDZ$`r{9SY;s!@G$h8$Uz`{aW<2+X;+McDn1U_y1m(68e~~n z{=Jc4p7BPS4s^Zqy+K6(qCI)EYai(fPtXqA8q}_c1l!T~TLPTsJlM#@jBJcnO~J2u0>-%YqSmJ1G~^!Rg~GpdOm$&#sd-JpUT1GM`v$^w zhYoy(N{G`ZH-(|HmeJ~kBuQdr;%BW~eQr}~DJj0)yr>&&Rl837SWaDXi_*vjz3vs2 ze2#E@0%NJt=PEIsb-+%35glUIm4Y(PPLrVcJy9b!l1hvTew*Z{PLZ_zvJ*Dm+6R(a zaY2N7nobG~?|wXKPx}LFtD;vOx)|N>8g+{w#@=7uk(Ts%_;*6A9(C|DKKTioqN+Fb zQ2g%dNpls|nY5_o-;!io{%8)UiAh!=wWV1Hr1!!2X?@i!9}Ex5-PF*~5a?FW8Fcx0 zUZTDA^XP88q8jyWGA+&{%*OFj#pRFdheO2OVa z!{2E%upT8S!tvvYVyxK`W;A%p*qv|bk7`C1%fX?65X?qIBk&B&>F_?lN}@J^no|g0*^uPDrdI z#(igMA-fwSD_@-xNvZ+KzTCeo@`bDAxTAI2UId{g*%kCo{94b};evcrB}VE|>WVqR zA~&NlxbTwAdF227dLd;02-yXl`GW-s?BXrB!cINzhm92$R*Z`()e*o>W+l-peTl;_0!j(;Jr{DYMZ<1YiduTBvxsnsq~%Iu1%iM6QPQcqr76+|);ePc9DWA-Yr zS=HxQn@M^<$tIfDYn@17`B!CR7;rI_jS*0FjU1CK^7`+|@Z2VkN=ItP5r$N!Y?c3E zYi}rjP)e?^6^E?q&ev=(_g%unTy!)0&|i6Y-2CDrg%ck%kgkS!aN9Yd*(L>NQibkS zI9|*M4qAix62wE|ENjl8X&%#C_4TlelS_5yGgr~ld%bsiu|~)E74gy?)dxDJz}>EO zpQR?sF2gY3^1rFP&)s0LiFWI}>(w0Y><<8s+isax0nS1tX&N#@|8nrzh94$6=##r~ zfw7gf8{=>wuJ_30>5*LPTc@_i|4%7X!40cUM9UdHEsIvJJdmG3vtf8RGFtxmF?PzY?f3JeN!rPZhl4OTGbod-GiI*_TB)R7>c`X^3>ktF zC#Eh)4bvTzt-9kKrUZM2oq{+24?@d_nK_KLZ`N_*`?-Z(@jB2CeQGJ+hhjbGa!NLB z$6t;oRml0wtT%)Sa!M6GY9QZmGL}jS>hWQgZs>HCw$nmKpjkoCU37cm+zn4@+FY3O z*Ez_+;=#WhTRM7Cd`Robdi!PD&3X?0w~y%p<8{M9pb4*Ya)iN`DIpM8LGRde)=oGn z)5D#6Mka=2IjN1bqN`Hnb>;trcpe3SehHOTm9n5S=#vz3M5r)+^jShKCb^W7wdDx6 z&n1u zI+{|mrt!#U1OHQ8*05PWw2FLTMgR^J?e#>6B4+WCUrW~yMdm-aD)uE3m8mEwcMB|M zSBf^<<}r&uKKut130;O*duFWHIkmP!%ywU23u-mHxYLWxdXXQHw*L6r>{#~cwN|sg z73WuyM$2U(yl(*H2OS$T6FL|Ro`?k|(To2gn@o1ek={5Ud1bR}-O)=CvX|bv2m0&x z&GMI#Bi3_r!7`^flb$Y_1h<6@ayx_hBEO+fDX}G1W@aMx8fh|r9cU;FEZXsP5PR{i zs}AYi9+y)b>b2^SjI$O;a@MIB^1x$ac<18HSwjN;a+7no%8YsF(ZGH6yEIgSdG_Gl zG**d02H+dy`;(lXBJm($a@dWl+C&8NG#f0D#4C#Gqg$-z+85>z zi0+OLR0!C-r!J=9OIySIrnE_u7We4A)mkIcJ6oaY6KgPkPd&SuhOe}8Fq!;3E=GlV zrUwF52a~lX^Sk>m8i-PFy831jU7tiO0F=qqN0T4rwau?CMo>8R=+*qnlNTM>Xc1>j z4n|<2xUe8W+d04dYc>kH9GFrrCIeLQ{DBKtP>Y@It6Wn}K2gec(fQszN~0-)A|je3z$? zQBi$KkH4os2Ul(w>&jmjKA4q!4sc2g*wwTnKG}_S<~$dKziNKJmn-S+MJuMDb0>%S zocUc388i>1bDY&Ai5XC#zm|Bgh3pmW*0{T`H4F}gjHZ)I16`)BizkZAj%^l7h^!J5clnDCam z{#@ZJPs$vD`CY0xw8!67=T`P*@s{H|r8iHu2FQ523`5Ye81O_qFqMs9p!pIKk&p%I zWjx|V&!|72ZX?MSW?+r~{h%N!*CGo2^n4&_H)OU*&#~!@&HvLh3C742 zE^y0*p_kc{trt|G@p56)osS8>z@%kdHn{JSbudF2hx zpmfb15Xc9)VQ|>IL{Cu&vWnLGv~>fhyKbJ*8C+8w@~PKUrH*ppRw<`ap=w0-WX}Q~ zE;)i{yZ4vzhM-&riIu&kmF&=l@m=-1?{EOw_x#`c#u$W-0==Wn7}~5%zBZ83=kZLm zywkGglDM}OPEm1R0d4=iM8T-(B*{4D=5(wVcK<*CKfb-qj5XMA*VnQy2G!$KeTCm+ z-a*Dp(8hATe|^(Kd#__LeuCC&_~aD!ctxk|Z{@v%O~Y=^ktaLpG*qCwh8&!CSgFHB+EfEp3pn zPAKr8cCiDTKOF!n?9I=Gvq2d@bBm9&Qg4l-3xLVkEUD6v%*MBTt@p&V~+S2i;uf&N*IPf*|;k(>+gN?f+ zbv6=v&u>eqMb(im*DO?@Vym)5R^Z}!;DK0Gwq@_e>5r_3rjb}T>^F-YNJ41AVh5Nt zGO*;538A-l-r_~!C_xk!A|npg_?S$VXkG*(SX6t{EP=A%u2asc?d+sMQMXM_~N1WHTh_r{dMgQuW{UM56Fy=1?S^wPA6 z_TufyV}B0Vx1_CjC}@4vlUE8}AaK5)e@=FNlc-SUY710|FW*4!aZ&b}C5ERjNRxr? zxcD~|OHhYLGoJk1#VSu-3Krvo%0A)A#n%{mBmLs`Y{3GXpkvz}^3*Nku zT?nyWC!b6xVQu>=5cIpLvw1OVJH_0bNHFLi?Wo!HRjt9inC58^#G#REv%d6K=NrLd z8-v-1DgSCrdHd)DAKVy0L^T7A6!!1mq9N}}cYM;L9un1bUegd8ALu|fn3YWPQT-K$ ztR4R{R!(CZbBdI2r<`0?HYa?D*NzK&V1UlweC_tS9R~Axc-hzsJXyR&-he%sySJU4 zX=EZev`9$jX}EI>1nG^}`3{?mQ{CI>Y1lxm$&U+Pzr|U`?SS$EYX;lyMLrMzA7l2EJ01B=0p5{cD!2j#Jx@WnS3q!;@f@&$@~Gr;L87;j^Ee^TX|cc4s?xcxL0A3{!QHe{&&e`ig88OwznsJ`v}%4+CODr;6r@!~ zE&KL>`xllg!E%$bD)dlbD-|cTR%Be05vGE?6Of3ke(L)-kZ8Y5z&s*r zFd#-2L0mkXACh&rpm@{*`8DMRJOB5{$iI%<;4M$YDmNi)SA&2fcPRN~jbdsMgWh9t z*wh3ucm>8aan$SalO1W8gv@ta4P{AZty&roPx&zWsA-h6Hx>PYhI=6jTYIYiFu?0a zr}tQm^^sL!rK(*t2c_a{dIvf6F-q3#XzO*#Rnu09d)V}p;Fp0K-1!qOXV8jYbJmcOUB%MwW{G%eBZr8gL-_ZPXP@&N@WFa8gg*OK2 z9m;%YxAElErNfh6()eY;#XKEjRlJZ8wWKV@B|$LcQrLEgf%l$wr^d&$jHing>Uggg zA!k3d*xn8y6Th=p(20Bt%PBH=j8?{2##1Qj-RW6}z-jtG2yL6f<&(Q!Mi2|Ip0D?I z0tT6CNI|T{HpNTX&p#*wp1V)gHw)iCu$(+t0>C{$)X3z)ISoFICC*sylg9o|Q;%It zXHmCWN+U&2VBhZ6OLVoS>h+U8*T;YA(T@=#$%&(I`8x%2 zoF{X|`s)?B9$$NC?VTrD!bHig9i;9m(XjIao|t<~#FQTAiZ@oLy(27Vp^VaCkdje@ z(*_^r7Id{yAj79-ZZL9EDUyq|qMCbFlC9fu91I+cK7I|qF%Qro08D`Vzx3+G`o2l3 zXZOazRLF)P`2NmIotHWb+I=9&MMaeS)C`XZ`4%@x`T*-y$KGi6vP_@as6B<=5L;=w zhA%sIs~-PuIv-GH_^LjP@{kl5P{+DQeL_*sFv54xtPK zw6}{i+X0Tz)4=$tNI~zbY1n7ekgVQuR0LkzBI;ZtrE$#HQZJbJr2!k|2d*t;b+MRa zeb?~JXt@o)m-h3XQ0VgDdV3SsQPlCV_qdrxW!v5{*iZS#$|WD)!gd5BWcN5QyP-

8O{{6`|(Wc>xZ3ia*?q-GS_Iwgtuog_klzqk_Pem~NBC>)a!hitVs`rjH8 zd6u*B+I9W(Bz)*{fAYj-F@}7q;4Wwpy8`<2?T#|oL=3QNulftHh91QkHy&1n1~GCQ zH+BYO1)zlr6fi=hNeHGFL=Q>?kZq^MmiE<-?8lg)wTdC)n*z$0XPuM=@#de@e`33H zK)&Y<2u(7v>S3GW{__+HDs|woGzwS@hq((PP#Oey2)h_GSUAfw@Giv}B@YSW)dwYf zm<%#SsMvETebsM^YV=m=RZ8$n#m%#snA^!|C(lI}7b{*){Ilml-!2v{3C(b`DX%Umd&mpXM zO`{tMCYZM8QQ{i38{WGccF7UWC;~YMwrX^q2q&)m%3{+2 z61#|fZ84EkA$Jcb_kjrBBkY+eZqh7B{GbXe*&!l~xe1NGpepxS`+>TSvJAr1)t(xD zkV+W*O-{wGtXa8^Nu5KqtkM2W(>Qt+Wum%U%x?QL2Yw(m@{=R8pQ_FOlYXTB=q()6 z7CBpYh0t7vSc>4O6G6Yo>@+V)8Di<7=P4$aw2y2#-ASO|33gpt8CK7!I?sn1h`=qz z_-AWHE1*Dnx1U8RbV5DEB6VeSIq=9da;SkjY4i<$$<{^h>NDt0vZ2`ib%?C&uBKcK z=S3b_*Un>L-@w%unG>^TPX-lSU8i;6mq3~39(5(z)QiO!j$l7ETFIW zJUk4+?E0b2Kwg#&Mvok)vF{H)q7=@x#CTiRMm78Nt5gMQHtBvh4qNp4aQC57p~1aH zqZJ%?<54#E)6yQc|0c!qpw`g;dNKdw(wzC=DC*>ciw=Nz0ObRSaWI;1;yv!ZVASd< z&M=V0busp@)vK1B;!una%cbFKU!HqAI9P>ZzBl>2`?L+=+en^Zg=YX1zL(B?qUYBh z270!9@fwD)BOEE}Eqr>^x$yGaf$KCR-TKo02u3P-NnUx<3avg> zL=Q&T6dhmFJrvEanzu}0@kja(Aw2pmKb51uKG4-?igh~N3i>wlR^{(=O?YXWN`$s! z9rK6q+qOYn7Z`B7(`A9I087~-HpP2DXh*Ld6_k164pWN#>$TCypa{n*G9Y--2Q~l zwQoL+kWR4BQ!IZpH(`229=LVkQW$uNT$w~$arJyC z0IvPcm6(Te3de>?Kteh%?8wLm4FEGJNs$z1ckw4Q+)z$lSTua-34D*1V$}%HSg?w8 zZ1;Z%f>&5mgA3=(pTRKlvd}RuVT8{Q{qU24!- zqe1NWJEK?N`;&$y-t1MMOi%?e_Jv$S1rQp&TEk-fFa(NRhbeh>tq`yc4I!Jd>uZoTNK0706#4uLXyH3^S zas1ZNlDNr{VL$Du`!J4l)Tt${E)yOP-zTR-#>gG*%?FQkpfb|oXkQ@ZT9NPN6 zDnOVw&gPTs`7RX681HirC1z2vLqN8@qy9wn&Rju%qgy#*`pqSM7W_n2);tb@*;9aZ zzDg2)fpG1#C=IY@8&iNi{HiOmm%Tqe@am6=Rbn{|l4Tpw=`Cn#5SO!Hfim6^bQz zmyfqNrnU(&FQ}A&P;CHhCBzDAuW+$3p!}CGXCbt zt>C{}Vhi9M9UrYJtEO1!S5}nrhLpBucWypKsx`=sFDM1QbGRwX{uf95I#pokJkFnV z;sHUbY!CJ>)R`^k)T+g255k47-1G|`xGD`B#<~@)x>*7xoe``4Jc+Da#qmu-kSeEilsEIhc#9VYGcNEsU32&csf<6e}t=Bc2BDCT;Oj zJ7c)#O7$+RH%g;(eJb3|A#OTz=Z2WCV(756@H%?hv>1E(rhO(DDS;&@?I`WM7zz=F zU$ZN#F&DSY#Dv=0{@WxK`+TB@Zbf^|heC95adrY22ndqsEo_7iCuNvPg^Ntdy39kN1$_%sqaw> z=!0eNdsHv(_kp~j0qYa5#pV_27_2ZK7{3*YjIHzvYF}u7QwM`0y@sN|;ix z2MkhO+4lE`h{vA>^ND{rZox#eBe8jOL!3imEW@o01)C-lpPFo4pQ^zUjf2bPiEgsH z;KygJP&!!h>9fS{rNuc!&hZ?58`Yl^EV@+hMpy`b5a*$4%d+Wf>|9kUm*%hF5>hwz zEXa`RXLxMuqS!lq>1!eYc z@+J&Gum2>jpL^b?K!f@Q;w<3!q>*HP!a!QtuyF-%UX$E>I&3U^hh&%CqAq>f`^QUn z5d7*DeW`FSc8ftR2Q%>?Z?ofab;#PA-}B|pyJyE_`T{|cnNomMoP-R&4lO)lxiA=DgRyKp9 zWATJxx&FHmW^QR-FoDY2?BD#dYy0c~?jQW8d_musk&(oI7hSUTPwjBo;s*&LtqsJy zT+CHDTYI%^oF>D2>u^~66P`sHL@C~-WXg`2l$+r%JEv-nAm5YGgp3geFQi2niJjuk zSjHdB9jOlYfgE;6yX+PVFTeS2ThG!byZSY4TH2wz?c4z)JqxKlqD|EKE+e+Dn>$1K z-%b`Z&yy_=?(3JfpVwU_yWNyDf({Y?L>tJcDq*SiO?yhnS31$L1MxY3pmc_&HzpZ>NLE%EiW_ zx#@ME+X-zdOL}aR&upzd2)Xo3!kyDS1syCI8c2+`r%7kk1M`1+$>8bWwb!SnLp(2i z#YV4MvfXCgLSTyPA)XPKzQ&=_RD7p>#gtUMa7PoQ2ev(6_~ zvO=_pv*N%*Z=OMujgHR4>HK@F179~MhG#pVtpo~4FAJJt<9E>5KEDI(qzkNP0zx!6 zSn#Ol8R=5Wl$ip#_|roH(~F5EL{bJ}ELkv_BX1B`?)mzTFArZKnA*fKbNqH=;=UtK z*AC`ufMM%8?;rEQo_K*~kKxU0?W0PO;0?SZ*`-Ty_g#DOpqz70sL}GJGb^=VI`K`> zGr-=UgQF>H9syr*;0f1AfCQ%-Ka*nFt_<2us= z$-3S)y#55y_OYMg8Ec8D%7$f0Dxs|*;hhVog=C0EWta-kxx~icVZYu7YOK9&%YjWy z@mm+W3R?b??E7#@5dJY*VnQQ0;f!G$;+xF2=dx+pJZr=ORAAs7zq zU-hr?CWjV(x3<+Crt;j1Rs^aWN5`15q2aND{Hl}dnUwCx z&Ez(@S2I7_H{QFjys28+x7oj-;++qLSviEjsyW)%C*DT6Yur8jl<0nw@oBNTI;l~M zQni#$k9ebMou2IbDmFXdoKJc(J>}a-agHc^V>9r32eNu$C&zn!?$9E4ukjwSDwq2Sj8(g(?-uWkGj|&4{fkGq`XJ28nMup@)itbaTpHWR_7`H`r#%@=@L%YbFTcV&P6BAa2&V+q>R^ zp7o$Z7`kQM1I?J-I=m)%FGLc)+9fla7lP;#F(K{o6Vtgx&;#a5#u#w@vHaqcb3gNGtyIDUIcOe)gKAIhl$loX+HYx$6&KQ>R!`3(rEp zFQ4p--WvwZQb?i=^7Ad}Wc=!%;hO?~mw5EfWZM1%7@z(2d2lC?AkiJ>*>DrB7%W6D zAGf|fEAA-7U7Zx*wfQluvJRh7EFsuuR!409#iZ9~I14xPg0ZyhGef1MatvoFTq`_0 zcf~M~3(@x>F-5HH3zuj0#z|Jm|BPP=wda{XxlYkUV>zq}Gw<*25oO^O)pYG5*&5|& zt!l+)GMYVEw3A&!YrkZk6Xq(bWVWn1*{83z#E;dg9=n9criZ^>u1LkZkJZ??RN8+x z<(&*XNUGB1(%rCe{gColyqY8Uh`1Ht!U$3@#Aj(Xit8~0bAVJS_3DxEB_ zd^NDmDkV5D5%4#b=rAEEU!g=N^jh2E1wG+yYeX{e$aRmdf4^}sci;cHnP9Rlr~R^K zllSY;7+^^MG6dW0|jknC&)UBSwIbzb{0q{G23(rHmh}5v_74&r zbY)_~O>{h@<2E+5KP``&cfa$jQ(1FV*2kgViL2dF65TBz-}Mz$8U7Whewpx`c$ZUc zRG5ri?wWC$Qb@wENQHdShntsgV4Lbq7G}y`u5O1)ke!6ggBNa=Y{K^jXmJ)usdZH$ zb>AW1Ew`ZY8PINQ`0H{<>LnFf$sBaW_IdQ`Mc45KPWtzig7Q^K0W%-y;rwk{3DD9} z(#9&IA@ViQol_AJOR*pP_C5$DzT1Y2WvjR;&`ESz z?~C7LyG5&fYwl0~dJkJ~raV*gDJ4q2G>GeZz0|enR6l0Kv%xZV)u!6`{`@yW;e20TMT#Xd`=Pe+lqSo-kd6y;xE9E&sifUF~QM0-I7s&6p4I{)fMor1I^}ga%4n+jDMAcEy+e z6#$oW)8i%f)swW-*5kpy-8~iCr+26rkUO}$e{nFx8N6i)yedo)(8_Ij<`wu#CfTI@Nx^tw{9 zOkd~tsz0UnGW|V>y8c>ya7z{qhDwyEoJEpkg0T@2k?f@ZiTZ?X$!|0A-K8a|feq5xLn`SCT8dG8;$|wuga?zIV?h4jT)KP0T_$7R zv(2q3ziQ|-9DP;eSQ<;dIQJ>08$UwyEobWf&>mM|=2?{dLi6`@dW$wk z4*q6rtW`^<)sF-qj;4-%qX+wM$toGGoFecy(zy7qo8xl$n&I#M-ke~_={Ry+p2B~c zOk}5^vXmCEDy6+a9=oUpQ*OBFkZb>()LF#3KV)PcucO)EZo%JI_RJ?wIWElZUybLX zUEn(e;3v)<1J3c-2>Yvo zZ;<;XZ4cmobx27Khie!HkLK+HG-3fC1F9g17mls1cI<$eP_B_KpcMYEqxC?pr|x$1 zYla5b_M3Zs3K2u==*fGBllk4fdhh*#ssp+^LG%C+WDxHLHBaTr?0B&H41eMU?wK^5 zERZhSNmvQHfWSDz6&p3Usn(V44=eG_iO1uJ`uNxBV(q@8Qr8gfJ$jN&6GVkSI}2CO z2=~3pdUZz2DY0MAHqisT^b6jKrDp4J?U@~0OLNZ#XL30=l|fU;=KwLo4lO+nIc`TyJtl zxShL`Bawp__9O&G%$tm#@6S#@maJZ@=vTt00$V!s`Bpq-yGxaF-PIxm{~EoL%C0ce z%XEFv*D(uCPO|As>lC|1n4xnFNpR^HM~Nw!B|Xtt5o8;u}A{2o*8EnUt(E6=iKX!yG=U zM7Z?u-%K-yOC*y&dY`xZ;gX z5xI$)5xQ!;F)F>KXZ00EAcNVKbT}jqIt+CegYJK?EqjA{dmoDUy`A3oy2Z1D=00 zG}wtaykyuxW}w}37ZkQLkbAQm{IE=la) zs{{|o!}H0$jke0$aQE>>OgGuEH~6kw0TcMD1^;mj=wdS2M`^mF`0!AgfQpMC9aYY# zu&jK_1!7HyHOA5`vGywCt0ZX+mINk^cgSd!p;>B}NPm)c<`z|U`CUKAm}gg}Wx?NIS*iJODC@)b#DSe#@TBXXsT!dc{WP$I!R*#K{bzWT}O6 z+P+1p>we@*xcMc6TAZ!6$ArFNd8T!D!ZCeyTXO#dbzJJuA zpC3tDzFkO@;!hzCiNb$ch!0xH*OP4yc^vO=efNGN)<{%Z(yBAg*A6M=Q$1tZP$l_d z+Fh-Kzj6aLBZ?EYQuv-)&Fo^>*cDM5)KcND3dt}~U87!K8k@aLq}*K#HmrR7B7-9F ztDd5X7r`EmAKdCtbD-Ie8zIa%eE{dfa?W2gAaS5AYOx=~p8GKtxj<}vGe0D!>X=J< zoFXY%sL^fe+6S+qXYW#Sl4wA5j99yv9r9DCj6>1#Vm3w0m8%q9CppL+St+5-Xt_k5 z##dB)f?T|yh^`&#*z{rgbmF+U2IV(D7yfk2uHv+D{7R&7Pfj=O4(+2k6Of2h7dV}8 zAaJPse?)zEG#p>t_v+me1kt-hv?O|q5<-ONC2F+j8@;SvgQ!s=x`+4X z^zOQV+f6p}PcGaJFA4q*p))pt*qs7%Kt!eTbXpb2G3sgEnuS>}U6BQzVy&N=qpThd z0C@P@+shgNgLc%TItH<*e9<_O@7T`4BT{gLRm zhW{Fn-32rY)FI=mOKbbpereeU@@i*Loig+$WmonyJ^ZVS-qKIaE-4*1fHwfO122Fq zvfDBK#*9GwiUt2bLghb=3RlaQ|Gi=%oEi$0l0b_EEdRCxdfZ9UA~KEOREc(bL}Oam zIDgU17ei~^KON&#@w!WOr6d-_)hL6U9Ee`@_q&mv(v%A$3`ZRXNC2%-vRP%F{is_N z+inIWd@`l`=@F%VU_7&xAK&hCr=0L{z}^nXNv+K38be*mSY3}ds8k&OlxvsKY*NCf zRZ`vCs6j8LmS`%G_^4#CjhA@VbAZ+o5X^7VxgK(o0_-fcqxE8TBOT#dvyFt+g`Ax~ z$vGbk7$#3RJojU=AnASB??ooPBCYLP4>!fhkC@;nzs)n=lFXdBri{JVw7u;&qTFhE zX2yJvCC0xJ|K6Xs91jOfu0?`>Je`H^LS8m;M$qvx#VDYlQm69~<0S+-?HMHSH4K4t zb_W>oI{=^0+Y$7XCmtM>pWnTLuWbQktf4A_UQsG6-vmscolRYPNQ(Rg1~viBJ)4jv z7p#$EHmkXB?K9hk>Wbm5rC2d_X@{i#QDf7vMGbi~D{hbe8e$_wB~Ox>ztw6J$L~qLdYr zOdIaWJ3Rd;yxy+3l*Rrd4P2@ozsDa0n3*#Ib1)Be)3%d7$Ki$NP(jAk@@Z)}yat82 zoCPi6z58=Sbrd#09Y9@3YJzg7{J{oRSwjd_T^CRWrblD8i=0rHUk&Xzg7NZ8$ITGb zQwYYd^ao2}jOM*hekb0yLs$lVec%q>bnokwqLTuyb~3{d?Z@(@5eU^<$An9(Yh2)5N<3)GyXvr;Zb#G*tu#>qi#|%E84_`k2XBA(sqehwcfSr0`0eLmg)jHm> zG@L)tirArCMeCdz$JZB((i()^Ipg*!c=zxthfJUy<(`T9w-`Z7>}@xXcQ&S;Oyd^@ zIr8-nT&#oiw-qh?9B9TbjDJQ>O~Kw(UlTo&EJzWAS1(SKm2F1V<9K#G+QWPIxkmWW z^;-D+kD<+;5B*pJaNp04wW2Tt0`>8-uvytAF4(;BhG+;OG~UB(0~HUcm3jB3j2)!l z?b%Wv|97*r?K=LI?d+D-Nwi*iycDQ+fT;}>+IU;`QMhMSOfogRTp4-?K2y^-gNJ#< z0UU$=y*12yrXY!vyQ z@LQmmw4UG1`+4IzxgflV78PY8AF`0H9=+PVkYUCF4AbRBaZdHj@m%g552kj}g2=*xh&WsvxQC;DW}`fe1n zzX`>i&X;uq6S+AYrys^$5SvNG5X^G6Kt+dkWIROjY0je_+UQTVL&>2X#i8m_r*h{p zCXp0*MdnV2%VIU`k?p78KDWf!Ch+PgLT2J6kwy^%t8fZjpl>CE^HEn^hpWTO?1H+_ z?rz#7jT3RTN186{CFI~3*~rLsf0g{2_}cwg4BJ8M;jxJdDf=fKb!mB9Qam!ri3u^^ zDyZd9hy61t;=|zVAh`jKf*yLlN>~L$Hm-iYWoMO#9N<1H|#7kTJpiZQ*0mE=o#v zknd4d`6TR)4crE@Z_=xGp+Q|wDQbCIoAY&;r0CBT6W(ErL_K_EE3cQQcb^Xuh%Wwm za{1{D*x=)VH-9+VKffX+8WZ)I>5I5LFiL~E-JaDEGsFZ3I~l5m*ZiOBUaQHp!!WIYU!GP#z=9T@WT;0W}qdXl!^ZZ@!?i9I_?t z>_3(^E>FRxBJ1XptK`A6V6rRlt}OWbM*BQF<}*Gt_g%73GUv+-5X>W0c#J^FEWG}2 z1P9q69X=*QAB1Y=4DDja5QAV!$2tGT?f z)cVL1lcnsO2hUnw4A%!PK8mkr?|&aSx`Og&qp!dsDYe?%27`ycByFAX0lHz?uoZLC z-ajMA#^cb~6&IQ`BdF@6)5#}4ltxFL>fs%-ymD8O@P45Z`!OyH`MTMk{sKG<*U5#= zMB_mtYmp{(>E4o;9IQ=-0l!-khO_Y>?6Ko>LFnl|kJ!F8j|x8RagwS3 zcH&!gr9Ts4|E7Otcvw_Rd_Y&@uuHEYc+COzXy^X8K<`R0$6UWUzSu!tp*xLr_dd~a zlPx-oG&u<7kUBBd+y`3?0y!VnzS2ya45>rNl01kTwdb!gTI(Rrx= zWVRuG?0`qBh8_v2?^)48|B<>0U4g<6`C5CwbB5L%c*bEEbvtgA?T|T4W$ktg6Y6r= z<#jy}EztBh{cuNSjM`~Qv7(x*6Bft&zD6N7X-lX?afVJmEO@ZCxgkL3E?u@I7D8Q5 z7>7d)i=_5Dm!8R3#UCKzF54OTntQgd#C_6<>SR>{rABAGzey0P9yzLq@VAV+0gnRF z+?o_Sg%y40|BpRDIp^aQE=0FssA|_Y4MHBxr&R8OCZ!~v!x;l5b$=@#kG5V>~2^8%A^wzr8 zB|<4QX>s>8zyFx7a(pdN5KwYJRYBH30x$ze$&3T40F(=0MJnV+m(o%a)LrG4N8ub) zhbfxZKT1oKj{m``)el(fySh#JLgi(fIAKuv=ucb*ZTNLR6^T|2@bL(4)iB7}eo&yX zK+yt|U>6yUwZi_0Cw1HsR(T-2-hd%nic&G)PRodA@fY##h45%uTgb{!O6ib;*OV4& zv)$Z|H49PRaSZtJJ#=-~OGsyLy{?VuZ@&?eZPT~bqz{bG?lwgN0hWSxRh}yhVClcP7<1aflWCIoJB_n~%LyAsDuS z>!tli$@|_#izA|W!(@0AORO(*x7kiZ>3!y@>UFwCZ`?n;-_*KHbfqh~%l~2IaajG?!8pQ<#ypil+TSCV`|u)9awNoO~T%+{R$G zn}WVO3<|c#dUS7~w2%aZ+4j#fmFR{-E3e)qB~j?TxTcIQnp`h1EY#rQEGfE>utpb3T#vWzIkVOH+O ztyLMic64=lK8AP_ZI;)PazO&rP|9TI&+0stJ{u+|Un-WLG&VMd;x4%6E*C9v{Gy)c zHNsgWieu^mTmeAcXh*obLp1mMzE6SHt>qo_f6zbDT~dEk;x;1ciBJZ%#i|sIWxc_| z;hwjR)?0a)YN`EK@v0sc2ZXWuP9sozdj=t zWDQ+qzQr7ib#%i9&kxbR2JGr(TKn2)Rol7ekl6Y6A&Ic%NYNyf)@Y%bD<#gINWXac z!uMrM?PfG89L>6>J~Dnigs=@IX=}6GLal$*eX(!)9!Rq!>v1t{lJ(DV+V*vQ#A9Oe zInj@=H}2gWYl;F(s#-?e3SnD&n^-cX%bB&8G_Q{F{762IE7QMXSIEgW@rX`SkRs%4 z4u9S)`8O}|Mb`%X?ta}&20F&}^wtme)Trej=J;WfDwlqp!~!HHSBg~e_jT8#Rc|G@ z;7I0(qTWyVipCopbZTaka7HJ)>{egiN4b6!4-2il?6smi#A~K{ruyr55GmDAW$_lZ zKT{ns*`F-8$NMKND>XpP*IL=C0zH)?elDc8CAIL62Eh|SL6gL!K=k;s;l))anRga^ z3KB_vXcT-$aWcVKNXoW1@;J!;Rs-9%RWlj1B6D6ze}UfQACJ8$zi(|H$^1gb;^Hx~ z3RfyQfZNctc<_tV^x)p93{$$bfGSk$SA737I8n0gaJ2Ti0p%(I8+zlP*&t*(r1lV& ziiE_8hb-OIHurJYh)_%5!>QllABXfAq1%5?6xZDMdqxvk$mKmxxSmlNviTq$(fa8z zEr!uE@8Y5~PBD08Kjl+CdjB>s!&ai;Qm=|L9>HgTtz^g>-!UQufT2Wy31YaJh5wOX z&i)to!*f%EK=WDPh!Nbqc9NZ+Z-3nOtI3%2-GP@__{UZM^ zpa1v{FrT~mqysev1Axi_MaJ|l%}d}+9Sj&*kdiYnQo}ij4(S?x7UNyo3;Vp5lx=1v zDQhk*oA_+)HDL%6-1%O18S>oe&OmmGCcEMDTuD4Xgf%d&P!GORMs*YPqJw7_NSrnv zLsfo@d6}~(>?rCH8=a8f(RiWq<-pAT`LH1|eI-jfRopw_m{t8M1@Fr2il0~Rdmdzi zJJdXXdYd^^Ew*E5df4vSTY`!=!~@syd_Mr&G1ODpJ+cNrX8Bi1(2zdjYI7XJ<@4q3 zEn6V5!vlk_`dLR`F|z|}su1?!j0+-hX+WLud2uM~4w-j7?v`0#`^X_*kFFEX07+5J z?F-_gtT2x8AhJg|o8~)MK^Fcx@M-LyyBbv6mX{;4-Q52&6 zC$72%*a*K?!s28b`vR))FK6#lADKFW)s_?TA;c2*5;D)C>apZmd=7hPsDCR*_IUR# z6(mI&e&O`D=3XO2EResdT*IId{RyaDSxt?*1vl~3x`pUDj$PF#^8sjLK)L*a)jr!z z;|;;!!zF>f4K7HGqDm;tWvQPNc2DIuuxcnCWhpV+m3?p{a5xR6Tr7A&)w}?^6Q0!2tkcjbrg23OpDs1)F6!$5T9b3eWH~z&}wO%9;I>|Jw0((Rx~~ zMPym^+W$GlIl5Y4zTcAX@W%=4fLn9chSkp&ep#^kOHfs*4kYY zBcK4am9Ahgx+QJRB8||~Dpzo$2bu-{h>NRMXUL>>O)Sscv zRt_9rr?aU(e|PqK@i9wHGhw!M4*|s$BX?R=fggv3hm6*At6sLhTo<>S6dZ>PfbO5b;|hsCY-G*jH5|^ zw7?dp5&aOwB_b^NI?Q<00IUL4o8A6#7jN@Q}igMWJ`#L#tp| z;cSy2v?@IQAZ0lPn7awoXzwFBNdE*>hWbn2=i*WQ1xvmIPGk8&4bCdGd!|L zoIlfM?(~9y;;Al>!#Qw`feb!kc@4UYB>CR3Iv>D)PcNz zBFtCLm-paO`Y{}MC=QFe1ed~zXQ`ehk1`M;2Vydz4W(<>HR0%lNxP7$L(=w;3&lpw zesvAt%^;lnEsx3|#>lc`I*fqnx6rXP<314J#sEUZzXL#tlt>JJ%ceV=fCO(utWLRxo*rumc`GOO;lbN{v<#(S`Mij<{XuF}rV)im@CU!&X zl-Xnk|Ji!0n8!%`Sww_PgGr3x7E;J-g;JiHizzmrJX{TOEhdd%%wfsVvF2xwo7`?+ zB41}vUe&F|PFewE0E)a`6wy6%qW6q*jKQ35nS90Z2T=Rco_g7}v zI^dW`R>5~3ky`B0LKH8~MJ;t9n!i-~ zv1MmN_k=`Mdbc%R8OjUoX*|X^IVD*W{s2x~x2VYcUg@Ufods z%C>sbyf5ul^ z=^H#KkEARu$eMBW4-}OFP`($)9sEoC5R@~>nqpO8HL9Kf<*eMYfe+P6bq7YeGsh)q zIPT}Y_KWLfp&oACrO@!*piM)o*T5Q-{pC0*;EIr)vvauhyCd@@bZD(9sV&DJ-^=! zz*yF(J2X-fG(Dw#8Wj~WgV^S82ctBTD|EdiXOaG%z42QHa>9I4)IG9Q*6VoRq67Wf z?t~ILp`_jJMB$Iq#nQMuxjs^VpBch)H=v06DMi*2$f>(xMOX-&VBuRX<}g>}IdnqY5feA2 zimbtk#yX@VwNc;AI&!n{UwUCGe*j-OxMN&nZ0QXog-CY4kSSUtUI3_*?1}SK=MRE) z=M1dMY&4AYwzcP=Jlq|~anr9i6Z|A6JcY-=oN0Qw$?dO0Rx_pBXBMVnSE9LW(kt4# zkEP{zpw{G+ZrTKp(}I_dDG^i0xv#*VC1l1pLp}fN?am9O4_`@lTk9QgF|2~`TM)zB z6h6#{QWj*zmCXBdz3$4GYWWD(YM6qNalJ@1OH@~@-PUweY4OIU<%cMG)pfB>k7OfV zUjNBqGMEQjv$S;t<5_?dUROnqeyN)h^dR;BaPn2i7HI)dI=Vfk3wO1=#@Hnx`2n=Td$=`&_0HX2j=>K{&@Fh!9fAIel zzw`k>D@b;%qb-}`|RYP=Sk)7AA$;z6z z!+t%}Zu;q}d@rI3q(wjOaxcNv1UViirsQE zE-})fY}gWFH2~+Y9X<3@^zc=Z8#j{mDqmMdEs{PvC81F%@yjWYqSW~JC~s=!Ygjqg zAi<#m;dm^LyZ&p>7ERbde4?1N6vYP}>XzI)kYxcR>d{>W%pD=6MccM*-4e*UJLqCi z+VNcK=vmW?7pJ16@27b4>Zz!eB9NgKR~`P@lgL(0idL11(!Ah$%0_oQvWHbYWl*5g zSK|uBA~GJCvOeUjxq8~p@#rzkMS=Ep%OWg2u(c z*=3U&o4v%t5n&#ar~lpx+dk7;e;9Z+1*t<&xHuYOXP-niSrJ-eiKCI`NgsU8eDKI0 z&xy`oB_vO*6~vJ_h|DU0`}f_GEu&W``uCS&N~VtAGk7h{8~{w;L4x2rWQtUm5$-6& zI+byMUN{&3hR*=XY)t}y>0nzThq8s&yB3!kMATEDX84?LI@8bQKngMxh}xq<3hYN9 zrbv|7ZRzcymdkgqF8oOX_VqPqe9>BmRiKiceL|y?Vl=@>ET4N)BQqd#mMFF`FEh zGgIp)^17ch;?S>rb6!I;~h)T?~p5T52#k z2uFIpwYB2~7o8Uo?rdB4%aq&o4WyoM!gOOFr~qKPMGK*wR=wJI4R6bw38&MyDJDL{ zqDQ6#iYQ@IA&w_{r}Z{yD7VRmKFRU6H(TsO!9HUw<%8pZw1Bo}j=#Q|ev%%r}~ooku+e^fmQqDXkJK03(_kl1%t{{HQ4 zgC>N>)t8Fq)P{(G>MRVTF*$FBdOiv3zO%XNM0=_IXASs z+SFfh?Jiqtbym3-_hX+xuEV(N-9m5ey*voVY_Ex_TM!1HmqG|%)x_J-^q5q(s$NRC zj`#nNzyUr;=2^eTmv{#0a=7SEJUo9@q<5sk!K`{0cCvdKC_>f(B~{d7VxCZ)daO4$ zzfPAlxeEXuj*eUu*SLu_ChQ@h%BHS`_?zzf{GjIhK7Eb4JqNz_6TX~^;iA=F#M#Wu z7?F(*J&`n%q&euPH24DI3K&aD(UG`8FAobET(JOgI9fQOc0b1+cStTPry_ z+(y$f{~W0zKts0{tR{WQjX|0;wEM&N4`10Aux0%&ev!y?g5B8~ZlHYfja8r~a|XU~ zB^(Xcgye=uw?9A_g@Fu4rttz1Ag-kaY5)w%e)5}?%ef3-N^$2ass}L0jL6ZdW#QW( z0O~Pb5g@$jpuYm$1_0d9DB<0-%XAkmA-0Se{_F#mU;uk z(@E{~ScgJ7mI$tE$m+_TrmyYID!-LyzsOzOcH{cE`GMjlHOS8F-zWSVjuI%>01lIW zK#BtS@pUmB)#1CB|Nh}a^Z)aWYoFM&^HbS<@4UIijOZ+}2YAvfoS_=)zuG={+oGko z0H%J9#yX4g!;mOM1$AXcY%boFw8u`CXB|x=OVznaFzXUV+#4pBkR_sVmJ~Sr<)wHV z>H+0P=w0G>cM_8>-!j>as_Ily5RKMH(pXTq4&sU`wiQqjbjZtjhAIFRR}o7p$xRW{ zJPed*swxMbczZHlgQ0xnYDWY9Z3+22g>?Cxvq$~r$J~TCM+j_v*k>I64y-%a&}Bh+ zN8LsM-!%>lvQw+*$yBT`L}$gi`jlvOrOwfH0xx)UI?@?-KPd zBC7=4(LZ;F@3josYNiqc&4P5dE?nnJ^$OG$qKKb}b${f+mK#14(3l=3Dau!Asx zbBu{R1HPn)ytyVkUjHxv5fT@daWf5rEWd;ko!w>lY|d%s@h6}M0KV_Erm9dC;-q2YX(|>$7`myx4 z5K#i@anZ=*D~dpeYKENbf2aT0fSEEpFlI2jcNMFVlqm(Y!~8eQe^pHlN8YuPUU)5K zbh#9b{n)iK7QxTme8rTe#OjAurHRzYBn!=qetV^|1#n&sT&@@E$VCakN(8>^sSX5I4<{+hsh(c*cVy~1 zaMw1Moj?&5U9{Tev;Q!y&*b6eC~Vp%yw5NJZyZU~i!rABJ~DvpB}h$s!}Z{S$UrUKf}CjVJ`*+990Zc1K7~u5!F^CPekX@g-1|<=LWvw4t0($vCh(aofXrKe)vEKP@RD@}ZWMw#p zEh?&{@dL6uEK5kkE|T#}$ayo(dpHTW-wG>AZ`?)pa-E`KYB#+3a!AnhoM~4A2R_SX zYcRf|KAZ1RCEdk4$mXsxXs$v$LG_C5Hsh4{@ue=Wy$!@TU#R@ur4!>h;jUq)?b>M1 zMt@()K<9&52{x}7$$=2X^w@IS+G|>JW5w9W^_1e*>tLJ8E185UD>c2rmkq&+gdDQ` z;E}5@$uF5Hfwtq~tO&lT>bxcMty6eZVFF^4I~iZj9`;ZvsF>U|;&f*CC=Uo`2F zi9#yxfCEjB>Jj13XfjpX`fef>)I*1K0pp)l71}SfXjFj2nm4FxTuH@|rpd#U>5-}HVE8PQBJRUNVdl%NztKV#1X|C;9 zQi*;HE$TPk{5^FO&@z^cpV3XS0JO8x9Z0y_>=~YtW?Kv^-D@uF(?8{| zSx3Z{1`$Xj-XWnCel&J6?As*c50mQNTw=5}qko#fxIG z=-9Cp23zlEkSUID!=pza-lNN5wB!E2LqXG1rs$t!4(aCg^&&kpZE$A@%u=u_fN^>o z0oTU@{8C}b>nS_I`#3JQ{lb3pEAIQ5o>9jklp;Ux>EdJFU)O(La%56jqD2(|X;KeH zCLde!+-TSmhpCi<%#7!V4wQ5Xf-jQ8GV;;89`9z58x><=WiZx51v9JkOb(tTmYt~Rf53zJH^d~BlKUUjRzeUl5I*@Y{ktT(4v zca21&zzGQr88TAX?*AQkagS)wsWTonn0kC*itxiSb{TI_daaUeYLZ2!Q*y z0<*5I*AScNKnhdENAVYXOuNn|%cD`x@W%!E%=}qfwm0=m6@=X1k)#i~vRos334~@$9fEa-D8HM1j$}Q?f9{-B$Ti0w_l@Mn7 zC`$5y-Xm>^eF$F6R8y)pM3Bo1=k(Q^D1pVi=r--^Q?PWQpsp9OP6V0*pJTS(jRa1j12+?B zjT1*)6^P{(i`4)rlJ}*U-=4l%jl=v3Nh?@f)ABRv$4k23RdaR8((~H+v7Kyeel|rM zS{W7LxIQ=8BPF-&dZaKE$BR1Umc}_ldL`Qr*yCFfRXWhbhG6t#(xs@wmS0EBTg7jH zuQ0^gL90K+b?~aJ-c70PU`|eny2oe2Nq-S#HKYOzbIB}{8{&mWxup_d_pl3xFxR%fPxM^n+7w@g960N9 zU3;V1`J}vs?T;-8{u9380T}_^R)EY%B~BOM*-ihc;aw8SqYT`Onx(5t&h_%#R z07D+cjuGd&7XO}q=-U?Rvj98i>9M|ZF7it21u*nUWFIT992qr`z+xp|YcULvO!guj z64@GN06|renUxBQL_?CqW10my0D@5)!qtjsrbuvcnw&tcGmu@PMX$eSHee>+ZN2k| zHAFrNCL(b?{{xxMf6oPTT-W1GrR&9V3rBD#&+%}dsXaYGCZH2(B5~0#x^FXzGN)g@ zVTr_4&XLw}wHH$T4>4Sz!< z8=qhlv+RAhl_1UCe9iG-@@&ur!OE`LPK>p!zoO+A7IZHBC-a9>@u#@5?2pyR{#H44 zc4A;8D|fT3j`kS5I19Ik&?xqUqeewU$yR5*&N%sc_5#}k zeG&Z-L(PBRqG&ga8=XoUk9aBRV4)W#CKC@9RBm-37qDHl>i{~|799_nDjY~Xd`JcL^m1|2*06F8qc_5dMi&vr^$1^>@<=ZK zAs68-2^f3@a>Qq5ex34B{cUI&tZB==Ko`7gc+Pr=9Pb z){Xq7@Zr|GcV4GI+wPttqrh-0EG?!Z`7a>h%-0IC8>*uQmOg(%>R#h!x-#Fj#!^0K7-Lc`UN1-EVCf+6xt1?P2T3XTd1|$QSA;l zU6E~@jz`K`dUm!_Sv*$LJJ(PkKBO>_=dSlVZ1Ix?yEk+`d0&H_Rf`8#Y^JCBXy-VS zSO;_L7K9CF7I`!+Jd(YySIuw^e?uQ<6*Cq>*T|ZkyUWZHC`^3IV;DSQS48Lr|tBd^u37AKsVz=Dmu#XaE5}4cG*h5&##64>In0d8!%b;bszgq&`L-T;c)cHh^Y%l z2azzb1F*{UP}t<#^>4qukzGN=_a#z*5{72Hh_qASbT7`*%oa`r)K1nwo#9HlOv5kx zP}@~$k~=mrHYL0}=wSfPLpqY%O*~QRYt@)Ua%sr=D;Br?8Jk*DLe39Y*$17rGTD*f zUA+so`K$HiFEO_MiGm)pDS8I{LYs;*3+3O``QqDyXm^`+BPbq?W&nabb;qS_wwYfm z8P%t3`}~jswg!=xN&3wb^8`ZdJX6$%Ln3+!YO?5uYb`q*4?CEky`-^d~x% z-;0SJ`EA~#CY^XbxIkE01Qa6A2KQSgV zO+SB8HBJHAX^MP%JV5k+Abi7=3fL59kb|Ki;bGb*4y?CB%;NkehTHtPxp?3Cpdb8w zsJAWiBTtNKkZNcJHJo)n&w6*Z%N=4V@M;$R0Q8OX8rV^bCv47or6!vM#D-uY-eg#D zc|C}Hw#kH9w&ydMG0nbRQgo(-l;dz6_=a78PbKEm86M=4+26^$$6;?7PB!Y)Z{E$; zlG_swE{N=}zu)BhEWhg4>Ovc7!#e7FlmXWy@iLC~{*-PwIFn1xlUF-pR=q6t)yBK% z+A4fMG%zbp*r1;^bo4hCb~obkdjEco2v+12P27ZfMxmD?x?Xj5u{hSszaSRqpAe1P z`FPY+;;Zzbg-ESz-n7{yoikb|;+PGziIc0Dn&@AFvBZpKzxpZ$y9)SAwD<1aR)gty zjQc5%mr+TIm-ez4YlK3B@Q-IDzb9BLhcd^qi|m)kq+id#1zyrf@AUNX&H&1xZ}mO% z#7S?K;#Ykw>s$MUOsdT9DXOY}V_G*@vUuBbBrd%J5V|dOn{Yr`yTr(pD#a8IWE9gP zGGo))`^9RZWUR|LWz8j}l^Qv|%~Z_m$0E1$e&d_bPPAFOfxCPHmLsRSiA?J9V*p)3 zB)QEG1CoK(_e$l_ni_?6y$Iq&>w%HeOi%~OYHNLL!#m=GFd``D;wc;Zp-3UjYCv$5 z02IVAFfqzP>4bU)QPffzpi}=YOIw=Hf_DNyi7(HE+XT@m=Eas>`*xJ0>|{k)uF!9j zgAmE+tvy&{N>>Wp2q(2hYBGF~#f$z@%BH327&QhyL{OQOEq3XOJP$_2BxebNi2U$Snp8x@ANF%$pbaM1>~$A^kL-)2(t!IT z{A@zVmdY)1y7p=hrr!F6McdK76bY2%Xtc zk6$z?u#hFPK$_3G_(!5SQiYDMjwG4{ew&yiioeff%(LE~kmUy)w7-1K@GkcXzVuEjwJ5z&KeSKUwcXI3b?6S)4`H_Tg3b$PA7lKp(& znxc6BR0Lb*A&GWvAhC{fHM<*b=%Y48KK;lBL#l1%L$q742bsKPr#|o?<&p`1XHaVo zWs`jHb!;WW=6E=YY)-0&idvWMzL967w-R-EJuAVH7tej8AeYCf)6|*XY}vhYNW6oq zyN;4wB(8gk(mM`fPToS9E8GeU)MT#b!Qhgqt~__NXen7a&G9K&vEWg zZX814hkI!`FY$3x0stW&f4{JP)nEq#d2_)!z%JMajhrl_QukwMprDv)9APK+nRHY+ zJx18i-v6to8dk=CbfSGqxLz`?@X)TI1D9AblD$Iip{_Rw<+OV}5Ik^sTvt$EQiZiS z^;H-G?q2W+77^od&b}iVP|WJRycz=`#sNUV&e7@T?$I5DO3QKX{*n?~<>u5E8&Cp) zD11A#bO$!mMeNl7k%KlcZpUZ!kbpM-fMpyOuvxJi20K9 zWuMQ~W%<*xs`0@NpaCL>?Ne#z?Ig?At?~6V-2v1iD|zwlJEOhr9IAR#9~*){B5Pb@ zigZXXD8F)ODfMlqEr3m)M$^bsy6*MZ)&{ZHHMbVonB5g=x3|$+05^wxH64GlaAvX$h^-*kV}|`us$S4r+5FHFso{l6Qi25+xGR!KYa<%uJ_~&ct{I@)J=0(>& z)d`0j!DIAhkl3K8%2V_6$RoT7w+l>E_Wrd+&|%%gr#jR!2wonD+=-dphGTD}n5bn# zpp@;B0p@(2{}yVo&!!!7?ovXbYvL0OIZ7 z9S}kbd=Mb&sln$t{pS0}H>fD0s2NHdAd&Qil~tJZ<-^p@S0el-Ki)1IPsBF6;ISIe z3p)I}ME93v%Zb%`vGdmyEUkCC8#Re@xyVudhFyf?$4oWV^}Pe5n|AifEk~}-v@|`N^k!z0fP&z6Sq5_X zD``g7B4Hd_L+~C>B4Otm&jd*^>k3vS&5Lf@r;mOi6fH|T{=!rdd|D1aPw8J-CK<g2#qr*(+*(;g5;! z%O7&2``_vr)G*09<6kgRtCGA=&ikafD|>uNBzTf(^I&hzRFUNg!?C!-+YXCv^w*(> zUANb2b9K9^V}NhVZ5awZK>y9`k%S@k+V5niJP93yYFn~j$#CjCk4K(0QT{(5eDmi(-+RvZlvh6uIB=I1h zl~|70XV%N7-yUK$_(=Ox-rw(|PIl(&^0Oi))fJuRbHfT?zJ2DL##bVrf6k{)HHibG z;b>EL_ruknG+gjz$*0du86%x%s=$bNSCV_qWcLs}?%O&UVigu9`OR99!$1z9RPAIp z(%9Z$EEmz`c`~e+l~oXRq)y%bY}3{y^BdOf+)ev#VPfSogEzt@iIflS+lgHtoeNfV z5&KQ}+XB@M^#KA{-3bqrsGTYp#%-iK_DF}_-_C1zP3K3Cg^==5arCZS1FMKqT&W`u zxk=4{X&tHACDGD!s9K$Djy*@Dcp2afS_HX{Wi)hJ$IxI1&*ph(BFlkNPL=e@iU0OZ zMMt!Gbt~_H4FGh*g-{^3WpjH9BG5&z+h``R6pA-D5*mzbe(nmh`zGl;Xkp5tCC-iG z{xcp?RY6v;cnqu_0D9>~Z{!gRY)=P_t6pXdn6)4Tc_px>=-OaV*t}2AT3`nW$bIn} zr90~q3WlAPBMDeid!Tn46OQnp1ltn3n$;?~lh{^V{7R0oLaQsw=dX5upvUxJ9Bj_G z@QD^dIAZDQ`)>BZoT_2GZqrZA+sGZWe`?B?g#X|4NKJ%fA!~eo`+p1nI?e@pz`|o9 zl}2LnC~Zg;NLsK`RM%l==mfd&ckpU1o9k+>F|wqjE?BVHwtTFOgUs_84($inb~u!Lup2RS3ansYR;ZRw`=X!PcLPOCRtcBVVq#w6RQA z?Yudca{Yq*NFm+`k4?H;3QV(jB_zJxdvjZ}^}yL|Ov5Pia-+WZu_t<(MasB#3!3{X z5F!+4wp;pEaFV z?i?Chl-+-JdcGxp*KVBmxjgpQ0{w)qm}}2zXnFbjD!;|Hl7+8;V*oG{=>gWW$g)*( zZE#0#peNn|f?8t_Y*QVGpLMi3-&!)_WLterwp(z}Z{#m}UqZ_EaLh=s@~&cvhn#VmgyaarPeyx3+pjiJ zrGDaCk&2Mz1uhInbm>``O>O9{tdUKd+JM`@6>X{mh?D1m(T){KVA%)m>towR%M_%h zAi~o^mYM01U{%V@)gTp{`(J=WTUot)D|o*!hS}Gsp$0}9iL*jQU~Q|`Rinw>dx}^_c!m<5k)KiQ}-XcJY~jg;rXeLD4E2Vc#cJ)%3O#Y zr)%*|nuwue>ZHCedjmNDS!P^AFboVlQRqKKKn)gNO;ud;hM8Y9H~G!<+hHUA61jrv z<)!h{Gs&d8p|o>_(tGD)odp3Ne}~r2<&-D{0Q)yUaZCOG{`H>xjb5>v2Fdc}rzYvm z2!CyeR00&_@@GyMOVEYvyw^IwtxGW85rxYBJuCw5*~Qok;Z%5PJKQ1U?>h0o^yX#qGx zXa4T(U+u_OBoU$jUYv920nLYuSQ|*_ZQ`}{SN{emE##d^z3vlmnb>lzK6K8%#x2*3 zM`aIR?Zp9fAmmF3sYsqYd4GA*P|X!+?^O`vfRu)U(hEru zz~{UlWhHied}__zb0$Rx zd177WcfHAYg2G0^(sHm;fs25&ZF0bYiDd<1w8+%gz(er2f5p7?B_G^Z6?fd`1H#SM zCwM!zMc92!5Z`_%>ia*vs;!&hd|r_A1>D=Po5qwQ$DY0Ru6K|w_SNHMo3-^29+xNT zp6wj(Dq34@wGXScsdF^FpLcU_<{zYq7Ia)Gv2osn3Viu>{sfTocSy=A07lg+)Zeok z$@QE6Jc)pAVGPjm2@D$(oiEfat6GI(Xt-_>U=9J1gV+KO_Q{dWIi6SWmSUr`jtdZ- z_5aGc5^$*3_B{wmND>l?BuR55git6gmLuYem`+5pO!jSTNt8WFwj4_oN%n}Lk|o(g z)*1Vr8S7y1e|}Rs-}(M?UCz}xM>Frdzh}Ac`+1)CW!}h!L2VO6|LD9&u>ggTKT!~c z^lw9Fx3#oUtz6AxboI_MVdgVNe5*H}@p0)1x9&@DF3K8LxN&j)@8g9jvx+nC3d7bl zQA(PcmT_=!q`FF{$w8E7b?mced-!gv6!bjo@Aht9^T?Il!^iOIkG~*(N zDCBqy=0u#`2+T=(a!@9}^`Ydgl%arZjoNJ7WgLtNGb3ZdcD_kBg<#Sb=Fc&m=f&W_ zgcJ^#waA8Q7R*;_`=VD)gcrcigP2ecGbPj?zD-s4-I$pypAY&|G&ME7E><5@=HlXt zIyMCTU*k5@V~H0PjMpYLy>!&^@@v@@nfA?V;gN^!{n*Bv+$w;V3>EMq^R9A>;Y z$+VvuTT?X|Qa=2^emFWSZ&qmA-$H*0960>%SNiY^gCUhYtGDWMil2HD zq7ls8dS5?V1eOc^IC03S7&`MGfXW-jZ6}Ew*kF)|t7VXguk8@^tR?t(CG1iS;{9>< zW`9o_g?L%`U{kOA{oc`UMJ*3`h;7H59>dH5GB!g!0{2r@!X%`n?PHo42;*Rf5KHtt z5q`n%Yg?Mdg4nVK9*&dwpOQ4r1*QQ<~mRBE3NBK-{Ewy9%OHa+g+8KoWscvWKIxGrt9pk;Yl7DXoc$i~Ip8g2Cvq z5Dp9j9IO8*%q1g}Npw_ZgOicWaHm9|fo!BC>v9C9*FK^k@u_)`&r^S#v#{Ch8`-?Y zNd~x>Y?^{2@k4*ThcXhn6lg*y(O-vwVu3W=LkY|lyN#ltOgT>n(@D2aom&0&P(J4L z3JHss^y+XvJxVWl_4{T*_ChyAQwVGP2t>D7){*4K|O*T zB#^^zZVd+(cs{d@R~$M}A{*1Iv;Csj{j}u;&ya`zeocft-ErgI<$ZyhF7*Qy%Fa_7 z!(EEX95s4=NuUuMF(PLiWbdHb0g60Ia)#UIIPlky00((Em60@ywzeY(UhmRWb{~84 zHShEDLla73g*!Hv2`bvMdx|4M_rRb*l@1LBoPtX$q7aKC!&4}Q(R5Fu9!=ba{!)E9 zm_x{Yt7xWT#oM-I*z>kJ@_xigdL}Xi@qfLpys56Pc0^THj6at!Et&mRmtnVZ!Q)3* zlV~b!2xuD#0#-8$4?FyGj1sC#u*q$-X4c$h_jji^4)h%H)yYTh>EJkqJ#3tFM>sri zDN>X=``@p=1KibAkNV^_)7~87?cW$K&X^?sxIliaj9y_V$?8YMthb9;yjWOnkENa`$`N>kJIUDY_ zTF2b4W5{B!h5jIa8)p-^&RBv?WBgCx$W>}Ak~@r`uJRv?!v#0?X4(_wYm09z2AG4 z_u%85#Z68T=!*{A8+B&8-3K3h8a5bH&J%~6co78`2SB2YTKtfK6dMU}CR`0O0nUbj z0!D5NOm#y5VEI0N0sl2MP1bsLt;D~c8FfMIen8g=seiwm-|HWlkDA)dp^PXlCs99V zzpu?N_B#He712!!QFyDp3R4UINPlHBu^bBT#{s%Bz=eng{QWR^vjZALz%jYavn-Rj zSY}g;UNo5Zv^Na5RV3O4_F85h60LoD;XZZAv!*t?@#%3m=9mK~4K4^8F*oJmf5yk% zh;6j-bLiIOzG;2vXBme!;9%^BWiN}N!z<43VhKGcR-U@NhXF;$faLmry_628Z0l17 zzPpIFvpL*kLwPOv7Y&E7{_6OKhA`6zI0}-&iArK3Lu3cnTtvD=A^kvDjQP^DeyhQ} zKE-10r6*e6^3rt z&BZBGy7wxahtuofiXu8V&nd>eKr*8J67^cm8)npA^ZI|s!m4kdgdb&VTUj-%Y|d8u zsH}g>PKvO^ZkC0@((;>24!ZnGI1psxkPe<2F7tR*q*WcNTZmC^g% zWg&{~Ag{;Pboa(GU$pIr(TRE5k-%{v@#=u|zb!>-k^a@e;v`EVe~2adjiGX-Fm`OJG9QJ4Tq&l|hLJvmR$ZeRtKR1ZZ) z+4qE^YKmVb#uw~*n60s3@ zMvpA|^Q3OWaOwV5oYm%+y2x3k|Ai#!rhg-6DxgT~SbQO+b9ui)(a@;Qr>5T30P`xE zGWb(?bR$kZ_&y1Y${kS>paDcX2pT|YwwKAlL3(h&0{_j+P#t8+ec?{j__*d0R8bIX zW74W#bnK4s?B>$8gX-+1GgFV%-PcO59SUc}A1+o%l|UwBS`HRHLAp}_ooF!kBcB2B zmRKJ9vqz=sDY%%b4g6l|!6NcUPdHy84M&n@59kbe8kQ}%zA;$qO8?)n2KlqzY-XC^K3oG(gu z6~W>m$P}UbE-SCU-|fWzgZ-|-yv>O0!`_C4o z(@_uk+j#Cqz;S#6#|EcVg!7}<`&aruMb6Uy|Ghsfyf&2Km0_o`sbXX6xn9S=dHH$i z467~&LUv&^AIt^x7C@J|wEQ3`M0LwTvZi)~Z7T${^J09ykm{#UIOnK!a(rE?jAYs^ z*Z7^E)^47|f1@K~Jczs{?Om#}jbTMC7nzXmZ}nqNw`uk+g7`X6hpZhHJ^%Q+9?>s{ zLJTg9U8^n49bU})%V(P7K1EClV zZAA}I$U5P$tWkf0B?|wPHtR}jU&oeTTB@m7o3of3nqrSI!_;rXOqy*_GP#@i6}2Pc zZKP26^w6-2D3vegGpv!u5#(4TAa>xG^QRGjX+~>~9aW_cJ%${AEnds$4KI00994PG z=6^H%>VsAk@ke)3&gv(Vj2@R>s(nwJoR7HZ@UKBg8lrcmQ=* zWi4B2SiJTW4peBs9#)jye`vrUfVj*Vkz<4^UteHxh?h7Y1TB9Ek7FjK0w~^*8%o30 z5bvRO*BOBH77C!LKTwqZ5H;H=ji{mkz6vAmuBHEt98!x800F;z`LY@v9liM8c2TEj zeQs&*+;GGD&rcvIf%`$YgHy)(t=)*$E5!I2KZtlk9UOV!aBT4VNU)tY$s`)<b)uPa2Oz)h#@f8|jP2NFuhLcqQyumeZNYS%W z2T;YeUk46acmk;)YS2+Xd)9Si4bOzdY7G~Pg>$Oz2_jrRi+`z4^r5T*t}5C&F4ri_ zPD(uhIFg{wn#eB>W7Pp+#}ObTQ_t`dT0&}oYqok!P=qA+g6zYo0lj}br@lEh6~~H7 z<0@&e{5O@jd2{meQ3;7DW|Z|8p)ad_eHkjf_H7yFHIhleCxbTg`C(T)SSn|Cl3`9G z{Q+$f@GUarC*wr|+ya204n?iCtrsp4-b@?|=Th~sl))ddBt7OJoYMMHic^7dZMY3k zR52@LstP$=>0dkqlBPbtfFsJACn!n57wsX<5mlI47s$%wqau-){r?vm^>{#geZ)9p z_T5Upnoi$BfcbX@1nR&scDHu2bcV{FOAA4N9 zws%%^`S@d{xy@B73twEIXT*2c$E9eYa`(92k{c=|<%de?k*WA$xVj6Oibvk%o{h7u zlM_HiW&)h-a4R*6w=#SNNj->x+NDOhYtG#gkU#0zT%4g%ME(~hLH@{|@DD<@RS_JADA6$--eSH&C4HVF zrVe9Ni@6U%+PhU1=*>MZJys7&b0cofyJpmyu)Hz%!J(C& zNGV?eNsA9Syfs-9-3Gk&gW7`#2T1+xDRO59By1}4PTFAr&Jv`YRutvQ4xm>Re|Ej; zz5D~yud2LJZ>4%cBfYHruTL)jr$iT@jO60bos5#0q$ zzD0w3Mafi^o=UklkNez%CL=SZu4^oE?iw8bnxf9THxC z(H==5T)q+stRCN@VXVZ*T-TjD{7R<8x{)D5`^6Fb|Vn)+sz8t$Wd&&1fa z)8J8q=I-Nk!k5nEAHC^&HzxS{{hXKN-3~ez=vj8X8N6E|Zm4^fGiM~PbmoWAd(YB} zK0;29UY1Kwrl(7B7WW_b4@pO9S{%OqICcf^<8#Q5kAlM|&Dw4soU+#rvupbCJYRC< zhkTRwss{rKUX_dKYm|Yn7Ypkudfk3RnMq8Je}7aO@xvD_Z*DC^tY_~sVxyGt3G~o! z+~UMwD|u#!)>`BLVWXTE#=r7K$2ONxxi^aVvYsTXv{)+%pxE{huKFS z;v)%s_%m7L4EL*rIur3X!{byHP}Z=72fEWNc8DDo&8W&^G=%1 z#)ggIU*Xu=S>+o&qruHCIfN2DmAj_&A%rIWqupSmMk2zP(37KM5j#;J?0=2d)0(vkMw z2+3Km&C{FA{ShG)?u+jFAh(BfF3`qy2PH+VC{B@L0PlC+67yz5t;m1@AE{PE-a+Y8 zF)sC%)wW9s?^pDc8qBia4kxA&98YNGTwT4rcww2FY}zS-<5HyL`J$D~C1P75nQ(On zjrF7zJ{53ub#s#%gmSs}=e2+TAh(+|6Hc+c5D6C_r6tpy%8dHGm&7owOGpD z@#kig8|y37)WVF3+qQ`wHeFU%ry?Y=3`soX7i3eX+Nb$`P4O*K+_R!w3g-vQZ!;6# zu;D}kZD9LnnedS;0xbuP%lRo~EQ#a36{3@tQ%-WWi9gcWxZ=hQ(TwQEzVCbIQa6@y zD;E`KyvhQe*SWMkxF}oYsaq*0o%|_y`8++|TvcP=tSG!CmqM8vx~Ed^M6w7QWrB&l zYi_W77+H*p9hxNIeblKyS8LyhbTuz~7yYlVfKwDbQT=^w3O5zr(a{pwt(R<8KY3Mx z4x^B4CPl0n3d)8h(u_==O98ffmT}|w=WZtNG6rwWs-)3$$ui|>KJEuTFs10FyN02#>X`eY{Qw*p5I5cA|u!kdi-!i_6Rsd zOYUycIW*}79VX8oD@jbc8ob}{&q$FA+*Sq3tHx|w8%{W({-l!k@{N$(3gfo9xt4AP zkF3$Gu%zSJQQG6YER+-hT=uW^X4?@jvv)12SH=kD6;9B;2iTf3uzJmu~>!3{Hk_h_J{-J)yYCz*Uk)uNM6x}-p!1yb^8jQGg+hP zWsG`nT4ouA30yPMP|23(9tG25$G7{U70o@hm~kS~3OeH|aD9xgP&gaTsxrogh!G^2 zd8e0Ht#6Z4T7Da-ycsjk^9i?{UjDpuxv!;S!$;NWZIy=gdH4-Cj_e^D;ymp;%Q@6` z+^OUd$ZJ?Q>$QU<~L!1N9pThpnWVXWJ@Ygwt{b8J1Zgb==Qo)+I= z&6UOf=?uon$|xHf~Tq4F>RT>t&BlSCfg zQ#a!+sqlXoFiZ3pFh_n$3ZFnZ+4OX><48zYzZ?2_bIW#$USGk)hsFxKXW!MQs^wak z-}}dmIv+Zyvif-g8igZkGRca(ft1y+GGz1Vuaz{P0{*H~Tm*cUs@ zh(APhDGu6Bz4|phBtG^rjBO)nL>MzpT8&=GI(*;HW69b{v9Y~f@WKhPQuPO4y+}^Ur z@Oq{V>Zxrwele#M5&RpL#CrOTba4|E4wuHM*8)B2#_8_k>xpy-T0>|eu8zX1$7e6kx_!Q5J}41pio zQ`I<``l*M8Z$7VmS1an~M{HxChE1pqk^kW`c{?Q_Ojd@_qAJ=r|2)t8Lv9P<`qoyX zL>}{+O_vw44TYiS6FeQs7d_smE2r?0&M8tLF+esqSFO2I3AT(MX51*`C538^KVzZf zu?Xl^68^}Y|GZv(QLvlxcc84c6lt~JoEGcinx%G=UT`KY(-SUK#!M42^>mm8h(^d0 zf&@z`tHcKHIfc`Ix?9;W%EkNoLh;_M^;Cu&s^;~~A64%scaPRiT@@RjiIe91aX-w@ z7v24#*pmSgVielLH!W?e7eK|_vyhh6;Bm1eJ`2JM6KgzGExE*qHR{wizCQM{HMO`f z0BmI}0%~+g#r|09$_fX>Rgh( z^?Lj>uUViq{_9%Li|4FZ{e+zrCSKyhK4xH(-1vvSknY1DQV!E^JU?Ya3^pC;DfB}> zqQKVkJ2Bx}B&b+pBo@2mw- zpFAiV$4YFiz*iJfZt2Z7*pt2cjaz%k79+1niougtQx%c-JvrFpRMk<-j$h!$A45Xu zi$EJux+J+&Gr)hAP9DO@s&xKPP6bE;pv{Q(mk=vtMiwKOSALo}C2wBO0y&DnK!is!9#ucDb(e9bf#Wv&6OabB<#nmIrM_COW}2+KIJ zS&Ey4U=?9}B3+Ve?Nc5E2idYx6xb0k3hscoDFZw|z|H8M4`VbcQO1M7bEn#lbYHCK zpsP@>vd27~+lwygd`-QZu6S57EhcXuN zv0_}U!EOZEfYTj8XvKrWPMX_rko(GfvCgVB@R)uOt!ef;s8(dZ6LF<1UId5#&R)^J znBi0WrV+iWqT*xm5==}r8a|4@ZTQUt^=jTVt8+sXk@kju!}`ruzB$QlxU4{%7l@&7 zag###=Y?8U@|?r<(Bs?LaA&JJ-}}KsM799zH_+ymv_h+IQ_qKDabmEAlMO57-meVO zvj?#^+wRI@K~nsyuU(^(0^{c@nSbqouV7Ryq@lf?Bm1+L-eJVo9{0b@3@>YpZxN3X z$}5ya0OFhrDVqt`#g2D6ud$ny{ew7954(UV^uY*Gd4I9Ziih z&B+X}vY{e?zKZTDRI)N0T6t$n#GdEsVAE+>P~xG70aNY1P4Ug*GIwibH|REB8$B7M zA@gG+y@3p?Uj_E8Yi-+F!8?m1j~;`K=vW+I?^g^q4#S(8>IClX2!u6EyS5fYj8juD=QXn#jfj;K z$|BDyU8k)_BW0)@P&WUmn^|)G=$9EW^OFHIY?|Ok;3GAS16adX`c$2%l#@!%7qR0~ z&{_rR7nHrgh&N||+#|$4t7KU1S;uMoX|W#`vTrSC&v#$6vbnX(Ccc^TZ()r;t~~2K&utZ1uJ+(-nfxmMKhLp2IOYD2 z3YF5TiXWtp9v+1Y)*iM4u|cPa)-R@|YIz+-hY7=?EJ<( z%lmeiP?vHq6MkcTUu&aI>u-?H6|Z>WiM7I*@xnX>kL&{F*Osapt2;2@=8y#F@qY~d z3?Et$i!^KkIFXca5_#C&pfb>+KIFH4H9+kM#7_71S&_h%|4LDa84s^ zQ;ELB;h#Asgx`7&p{LUA<*t6l?2*k@I7k*5&%92aryT78cl?P$pBG@KotHRv1|bx zKrDc<3g%gINDKrrBh6Xyy=CvwyOfuiLhFA~ma}Fd=jSdlokzzAu$=M7_5auQEm;Z{)Kl zxLf?s;HhE57-+M|gwrFNA`gY6WrJX`3|NQwg{791lOC?`^1DCqnmudUT8QsdSjrLD zkkK!SR6`#_fo`vt3n$BsaB-%Rwm}O{2&i&lL?~1mE#wL06_)yezE|&wxHWvyY2X+< zlypRVL4wbDga&`6(RPG00;KoFo*d*+ycek&n6yLSY^LCa_wx?yn^zZ#cWylqVrkpD za8l52c0>0|9NcmR_$y9vrs7uHyWlHuleHLe!T$<=5C=U1fFX1`*UE;`^qB)RLiQ7Y zYa#+8pkqW$f~11?J>5%x7ao04Sm3#gU*l)mTEN{t;qxpZx!bCF zC3q`5zrGJ{-}I8&lu-EdI*)?>x0jO&<`OIvbd4DUW;fXr=&I?!6l^`?uPiW4oTtfq z;ICU7ZB*f!YlVMw35h#Du$n-vwpSUmr=9o7 z+S&~i-ubv-+0`%D%(&I+u%Tepg~5AVPH*2GOeyrkPTgm>WWWFg(i*=YOL7OT4IxCp zHR}h~J^CQltvGh(^hz=X^KeL zJ4(bXvLxmYP>1&(wRic`dEdIl(|N;BrOFpsPTgSLTFBklH~z&)cI^2|=q+1`Z5KIBb!3FpvZo_ra%(VMM=h_a069(Gk z7*%%K*LHjf0mLB0MYi2e`7CZLptG44c~oz_SEMfKN38s?oSduIBaZ=J$*ooRGNoLm zv+W!bQlaLHI?cUc#I${2wEP%HM`%Q1=$Js>OeB0dH~us2F_ictO1KMRr~`lx1Q7Yn zqqc~d9zP8r3&IHU?R(z(V~E=y%tv_@6sRQPPS5CYV5mz6`nPC4YP^kmE(*T6hs8vq z=X2^{pIU&ahg41&`>0H&i{|)ru|hIXagg|a%AQ|LiHNV_E0d0?AZ7^KiWSX3VdwYE zLUIEI84+T;E9JmsL5R-gQa~sWE6%j66F?hi-WsBf>|Y#Z%i`THPb8WoC8Uk}%>RZd zsD-HpUNh|shoNNZgIj@@@A0r7yPX!ZH{s#_TI(z5a0s5%cRJ%=+ryCU_wXa6B21hB zVk}Ljy(a*n{Fkf**x8h5ICaXrCyGwxFp`oR$}3f}_R7;o3TyC7YQA9oTxVOKpgmGO zj{O}wQP0wjT)MeTsQ8c5&FOxziq!=oJ;#jKoP>vXg;W|h=~kc(L@a*y07T4!gkIza zEdD2|fG;1%rND%^fGr33g~ZR$4NM2{2>^o&^mv0Rv`az2+~Yp|>D#HNE!#5AHJ0X< zVSn2p>Oajg&Mm=G)qBpBkh34LDux|TBjUxhTm2YA1uTHVh!mv+Aq5NY!etsQ7C}Vb zt`Cs5-4&6<(xzbp>)6K+6%4!$1AOiH?Fhb1XG3Zy@BMQ4NP680dk=Zu);-FY)hev` zIao&aGdRG>q5{RC`gqr~J zhj6dXUIZ6{+yl5U-E_ba$-tn=I0}RDVfF_4`nV%@eQw8ozq_V+@t4|OE>1i_yNG`w z^Scp#V|moLJdfpxj?)WGLj;KoAFqkJ^WBTz4HrIgg&u^ozD2$WjbP&dKP{coKuTdp zBy-4jeF#IJ7xvafqC1#rSU)pWxM?_3&lvcjX=9+sCl7pN5L_9gYCeE|~)w3+GuRp637q)2Hizz}P; zQBMix%H#nu>t}T($$%Bdb|JsxSWE`Q%%#vmE47t#DI@@WLn0gxBW{Jsdu*Wl9@yxF zhM1r;Rikk5w71!U+?GY4P;Yd1=0RYp;=gpdh0;;4Q`pQ}5?U~p+g339Fg@s3 za=|(?5CCdy>3KaM_k+T+MMZ(y_xm<^`uSutQ$ZuttCX9g2Swq%i1(ONSUAFq?VbKi zmL_%hDmyZtTVE2J!~d2IPrPD?_oDYpJ#l+z3DfZ6ahH{(RJs?uTu13P%3HE#VY{K zLV0TGWKNePoeC#fLiBQBQgg2}J?eVp#x)g%1h1*l4`l?a-kB z11;wyzDRm`&h zS03uW4CGRE3_G}M7RncvWyQ{_fuMbDr^wn8F?^7SBc-9BMb#hQe#O>DL1CoGwVLc5 zxW!^nM1^lMxlvGSiJpiiT$7!M`7)59=v^@@#WCagO-oVeRcc?j%sD3=L~I5b;we(P zsJR3JN+pO!;Lf3lHjTtn5F-Jlh&dfXL^;CXeRedl2OtG2<#7<26?rNb{Ms}=H|Fb4 zpVo+ewXi1dwUP7qx#>#c8%$r?xcjkxc4tG>c~gskk0tb+TFZYl7Oj_oanDS2QF3_T zqSU&pd;PZFeI6tW5}x^P{#`Ws8F|o?ujR-dW1hW z-gsF%k?}=8g;9HULM#DubzvrcMX7)G=&C0Qc@z*EsLUjjdj08@ahDn^N z41OADe2bM{uwIYCoPg<32d0;&3Ky$-hcP+z6REv>7A4}&guMMHsh~ofKYs+&T;F(P zAMR9=OZlAoJG+`s8MiZ9&v@=}{SLY+WUlxAu)lC-b{spUOp+=HqT8>|2D~}5&o!M{ z1kwXQ7Dj5&65A;Q!Wl{M;hC3GOS`=bruyS~T!gZS^tKO=E5d!IVL#$uQ9z2*16 zqXg=}9xv)ssp?A>WzyqkOxM!;!aaJHjx-dQHClOiYx|d4o9ca^rs+MsNY;Q<;lhjy zh7uAFr3*~J6tdl;p9EV2sVI}A2X~RqablM5Yah7&WYCZ`QBwc0!}CiCJl%WQLTus` ziG62vD;!Q4{e#k~#WgRQ>&CAXmQ*;}J%`hn3?165dB&15#zqs8XYzFdT^i=^J2Qe1 zzEQXz2@?pCc=juP0vc1V1XN9{p8wo_C?2|=r`~vX>V-VWESI9K)PvXb%uMeL^q(3IP4=cJGOf*Xm;K|JgdOD<6^hed#M7)H)1yOL{HHq)$>)Md*!WSnPZVLcf&YPWFq7x z0cbHmOz=B`OJ68b?jdp#B6WfG`j7%NUbkxYp#UhIpuBh~QqC40lH@&pfqdnf5wqmw z41)G*i(mI^<6xhUYkj{15-Nya_)_hcFOz-i7=mM3y&-!tR$iWzJR@p#_UI+s7q=!B<|n*13`#P>9gLS} zH4F?4u)>#!o3j5bw2Fmu@ln{ywQJW1CQ4&EU5bJ(6}jsJ)l21$O|4Q4qlePER<)DI&MALhBt{MHI z|77yT;{t=saiQcv?~SFr4L>({SmO{rctu9e%r{jGwaJgytQ43TekH~jL| zyz|>;Ke>1RauVjskut05&VpO#1!kVLeAPpIqN#>siayEmhAo+Aw~-VT(3r6B+%`FDZT7>O%i1r`XmH(F1e>R$)=RNky1GRF8xfN|8@1y(j zNDcPgwBZ3iUJJ+HrcC5dnUb0N{8HZPZu|WaF6CaAaj8|O`2~+|g;c+y#@lH#ZAz0% z6N2qaB1J%@&O@RFNt>U-wDpI_zO)F26Vb34I`O+%D)5)lbgR64x?ug{vA6QkOY>J| zZmcfk4y)L{pJJBbO4$e!+x02f^Y>CHOd=`UXk87LCpT*Hn2x+^YFKROms_T`m`F2B zF7ZKJJ1=b;+i}J?L*`Yykb_2%)_kT*S>^8sGt20$kJN3-FCkqDpzdV>9`zMkW=V7s z2(&Sy_x{u5yUH%{iCM$Tt7|WDmGVbteV*=R-~K9ked6}#O42l4@sU@$KCh#YcD}Io z-%zO1S>)&_rnk{>&&UI-;vDUs^|#nM>8Opde3Se}lkf6V`)%X* zhM&bu4-lEP$E!d}17MCc5++_%I*p`z;H@!$k`Ee#p6OD_k$G#~e*lilyKAq)5zTSx z@>PbbnfuaA{}bUcm_NCb@R+=Am?8dXw^D3WklSxIh3YJS6PqYuabfSR&3UijYHCtU zhYDrH2T%Qj(WPl;%c^*W)%ugR0(&F1wT_8~fo#3<%zmU9&$F=?UiIH8Mi$gC}yjdQl9NL|t^xN8fi+-$A>z8V~l)TdWTrB5?brsW-`rg@=Zl+4= z8Cy+#Gw43it}!Jh)BNBfTY>$}sM5b4$>kwGRyXtSC?KKdp)b~56{(-1X$+MWK{heM z-e@yF)4jr3A zofHovm1Nz%KR&eO+B-P9tq+d|O-E#(CORU_=qDrlS$6gVWP@ny1|md5Hg@&B>^tt) z2T~NiFv(`UVCd6--mlv3ub;`MIwMo8=XkaP?Q>1JR_{a!!7qds$Cqj{4Lkjerh9)Y zcBtCc`aSqg%P2I&cctM-f0vGmLWm+MMOif{a$3v`}=R=q%3O&j){`0x7% z&LPFBLFknr_LTZJAq0WIMd#BJaVG8N-2>O7Ik!i1%w1VIfR1K!XnVXf<%QZS_XdH5 z*%B)uq5XZ98lMaB=-wK6&G@@rNu0ke4mB>W{$!U-ITx3$nE5ekOJ7)Tq#o08pF5j7 zyUT4I#y^c^w3aLm#lQbpUOX>Wqt!I}&dMaraE5AS&>3p0i~In6axFtzU)}whf|m^) zRWqt5?`xlzxO!{bx6BYEn`D4_f+Xy`9H|Fa5bkR7yf3OU`V1 zj;Gs(M0{-=h24lhn00vL)@u62v7$FZW9~j)+K%-=*}6|EU*IqGelF0X*%V@xW?6l* zC9XwqF*diK)SdA3t+mr-^O^Cz&hT^98j6$va-fk3QYgpoHEaWvsv;4}L=;vl*>)3fYZA`;fG zr?{H+C~kW5)VuKwmGuuY9mW-2!i9FRd#ng2lr^WSSFPF^77G;K`x#g!C135*n#^R5 zQ|K_4b3Lt?F1qmb*;7(Cvyrg1;B8&#Vjq%1YNnpYh->X}iqyz+uB)HgU02nvx~?i5 z{Xufl@BML=$Ks^!xd-uI0#CLM_B6HTDF)q{{d;L7_8UCGmrXjG(;V&-Ndm_jPF<7o zsci9FU7A@df3vCnIeODb%J7O|K~2#=v7&M@-P%PgS%Tgg-*1r6F1NRT{X=cW0|9p` z`7e#-tKqA&%dR@<1lyzwn3-p1@U~n*MWJFw@qL;S8j0!Asg2EF))ajVLhekvjru)Z zYUe0$Ivv%w+bFSbx2aLE73{BT?b4=kD@irNE7Mw)@*{`85QLt2wjxjU-1e%e8)4i@*K(!t#rx zwcdh3Z>-zM%crC-x_HuLB)*pRv!>u<&^x7v-h>vVbDTcvtdasPna7bIKJN+3E>cHr z-avNbi`b3YR!yHYi~B0qrORZ>PUVZ1jccPNOWQ67&n~nMDwY+Dv!k!FE&kcwyw^x6 z;r61*x7?#qlLp~`tCapoU;c7JTYz_c$g;y?)7*>9pkgX#_0cvy@F?h#+cwLVRbT$| zjZtc)m}!lfspY-v%ZN19xbN>YcE`qjPgkO@r3S9w?tfh2$N$8>tzwj&SzE62aRU?aOOcpD*N$PI3nm6sm>VU?`g**Z6H&`9}o zYtc>h;i|c>&0-P{0yqC!B-LKdIx*1|9qV^5*+&W6(X5<*p5y(kj<>IkpL~1$`sRR& zPDpOQuHw=u)qnUWYH30_ZpxvgG(58(n%2LqCh6`G&DBXth(~-e&S5HI_?ys9{<}R| zu56N}W0oRC!}lEx!{?MrEYfm37GHK48idZxm`JrQcVv3Tt6X!(&ezn>T5a5JSCULZ zS2Xc-Al0L$#O9JLnKGw$v-6@NR3XXo`b~1)duq}#{sB+xy1RsaQP)Czp82~R538b^ZR#dF%~Z&YvG`# Date: Mon, 12 Apr 2021 11:50:11 -0700 Subject: [PATCH 51/92] Explaining the state machine used by FlytePropeller and Flyte (#903) * Explaining the state machine used by FlytePropeller and Flyte - this document helps to explain the various states a workflow, node and task transitions through. - TODO add a small table that helps users understand what a state in the UI represents Signed-off-by: Ketan Umare * updated docs (addressed comments) Signed-off-by: Ketan Umare * comments addressed - image for nodes improved Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- rsts/dive_deep/index.rst | 1 + rsts/dive_deep/state_machine.rst | 78 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 rsts/dive_deep/state_machine.rst diff --git a/rsts/dive_deep/index.rst b/rsts/dive_deep/index.rst index 6e01c1dcff..9efea9094f 100644 --- a/rsts/dive_deep/index.rst +++ b/rsts/dive_deep/index.rst @@ -40,6 +40,7 @@ Execution Time Details :maxdepth: 1 executions + state_machine execution_timeline observability dynamic_spec diff --git a/rsts/dive_deep/state_machine.rst b/rsts/dive_deep/state_machine.rst new file mode 100644 index 0000000000..e04872db39 --- /dev/null +++ b/rsts/dive_deep/state_machine.rst @@ -0,0 +1,78 @@ +.. _divedeep-state-machine: + +################################################# +Understanding the State Transition in a workflow +################################################# + +High Level Overview of how a Workflow progresses to Success +============================================================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBSZWFkeVxuICAgIFJlYWR5IC0tPiBSdW5uaW5nXG4gICAgUnVubmluZyAtLT4gU3VjY2Vzc1xuXG4gICAgc3RhdGUgUnVubmluZyB7XG4gICAgICBbKl0gLS0-IE5vZGVRdWV1ZWRcbiAgICAgIE5vZGVRdWV1ZWQgLS0-IE5vZGVSdW5uaW5nXG4gICAgICBOb2RlUnVubmluZyAtLT4gTm9kZVN1Y2Nlc3NcblxuICAgICAgc3RhdGUgTm9kZVJ1bm5pbmcge1xuICAgICAgICBbKl0gLS0-IFRhc2tRdWV1ZWRcbiAgICAgICAgVGFza1F1ZXVlZCAtLT4gVGFza1J1bm5pbmdcbiAgICAgICAgVGFza1J1bm5pbmcgLS0-IFRhc2tTdWNjZXNzXG4gICAgICB9XG4gICAgfVxuXG4iLCJtZXJtYWlkIjp7fSwidXBkYXRlRWRpdG9yIjpmYWxzZX0 + :alt: Happy case for a workflow with one node and one task. + +This State diagram illustrates an extremely high level, simplistic view of the state transitions that a Workflow, with a single node and one task will go through as the observer observes success. + +The following section explains in detail the various observable (and some hidden) states for a workflow, node and tasks state transitions. + + +Workflow States +================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBBYm9ydGVkIDogT24gc3lzdGVtIGVycm9ycyBtb3JlIHRoYW4gdGhyZXNob2xkXG4gICAgWypdIC0tPiBSZWFkeVxuICAgIFJlYWR5IC0tPiBSdW5uaW5nIDogV3JpdGUgaW5wdXRzIHRvIHdvcmtmbG93XG4gICAgUnVubmluZyAtLT4gUnVubmluZyA6IE9uIHN5c3RlbSBlcnJvclxuICAgIFJ1bm5pbmcgLS0-IFN1Y2NlZWRpbmcgOiBPbiBhbGwgTm9kZXMgU3VjY2Vzc1xuICAgIFN1Y2NlZWRpbmcgLS0-IFN1Y2NlZWRlZCA6IE9uIHN1Y2Nlc3NmdWwgZXZlbnQgc2VuZCB0byBBZG1pblxuICAgIFN1Y2NlZWRpbmcgLS0-IFN1Y2NlZWRpbmcgOiBPbiBzeXN0ZW0gZXJyb3JcbiAgICBSZWFkeSAtLT4gRmFpbGluZyA6IE9uIHByZWNvbmRpdGlvbiBmYWlsdXJlXG4gICAgUnVubmluZyAtLT4gRmFpbGluZyA6IE9uIGFueSBOb2RlIEZhaWx1cmVcbiAgICBSZWFkeSAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG4gICAgUnVubmluZyAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG4gICAgU3VjY2VlZGluZyAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG5cbiAgICBGYWlsaW5nIC0tPiBIYW5kbGVGYWlsdXJlTm9kZSA6IElmIEZhaWx1cmUgbm9kZSBleGlzdHNcbiAgICBGYWlsaW5nIC0tPiBBYm9ydGVkIDogT24gdXNlciBpbml0aWF0ZWQgYWJvcnRcbiAgICBIYW5kbGVGYWlsdXJlTm9kZSAtLT4gRmFpbGVkIDogT24gY29tcGxldGluZyBmYWlsdXJlIG5vZGVcbiAgICBIYW5kbGVGYWlsdXJlTm9kZSAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG4gICAgRmFpbGluZyAtLT4gRmFpbGVkIDogT24gc3VjY2Vzc2Z1bCBzZW5kIG9mIEZhaWx1cmUgbm9kZVxuICAgICIsIm1lcm1haWQiOnt9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + :alt: The State diagram above illustrates the various states through which a workflow transitions. This is the core finite state machine (FSM) of a Workflow. + +The State diagram above illustrates the various states through which a Workflow transitions. This is the core Finite state machine of a Workflow. + +A Workflow always starts in the Ready State and ends either in Failed, Succeeded or Aborted state. +Any system error within a state causes a retry on that state. These retries are capped by system retries and will eventually lead to an Aborted state. + +Every transition between states is recorded in Flyteadmin using :std:ref:`gen/pb-protodoc/flyteidl/event/event.proto:flyteidl.event.workflowexecutionevent` + +The phases in the above state diagram are captured in the Admin database as specified here :std:ref:`api_enum_flyteidl.core.workflowexecution.phase` and are sent as part of the Execution Event. + +The state machine specification for the illustration can be found `here `_ + + +Node States +================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBOb3RZZXRTdGFydGVkXG4gICAgWypdIC0tPiBBYm9ydGVkIDogV2lsbCBzdG9wIHRoZSBub2RlIGV4ZWN1dGlvblxuICAgIE5vdFlldFN0YXJ0ZWQgLS0-IFF1ZXVlZCA6IElmIGFsbCB1cHN0cmVhbSBub2RlcyBhcmUgcmVhZHkgaS5lLCBpbnB1dHMgYXJlIHJlYWR5XG4gICAgTm90WWV0U3RhcnRlZCAtLT4gU2tpcHBlZCA6IElmIHRoZSBicmFuY2ggd2FzIG5vdCB0YWtlblxuICAgIFF1ZXVlZCAtLT4gUnVubmluZyA6IFN0YXJ0IHRhc2sgZXhlY3V0aW9uIC0gYXR0ZW1wdCAwXG4gICAgUnVubmluZyAtLT4gVGltaW5nT3V0IDogSWYgdGFzayB0aW1lb3V0IGhhcyBlbGFwc2VkIGFuZCByZXRyeV9hdHRlbXB0cyA-PSBtYXhfcmV0cmllc1xuICAgIFRpbWluZ091dCAtLT4gVGltZWRPdXQgOiBJdCB0b3RhbCBub2RlIHRpbWVvdXQgaGFzIGVsYXBzZWRcbiAgICBSdW5uaW5nIC0tPiBSZXRyeWFibGVGYWlsdXJlIDogb24gcmV0cnlhYmxlIGZhaWx1cmVcbiAgICBSZXRyeWFibGVGYWlsdXJlIC0tPiBSdW5uaW5nIDogaWYgcmV0cnlfYXR0ZW1wdHMgPCBtYXhfcmV0cmllc1xuICAgIFJldHJ5YWJsZUZhaWx1cmUgLS0-IEZhaWxpbmcgOiByZXRyeV9hdHRlbXB0cyA-PSBtYXhfcmV0cmllc1xuICAgIEZhaWxpbmcgLS0-IEZhaWxlZFxuICAgIFJ1bm5pbmcgLS0-IFN1Y2NlZWRpbmcgOiBJbnRlcm5hbCBzdGF0ZVxuICAgIFN1Y2NlZWRpbmcgLS0-IFN1Y2NlZWRlZCA6IFVzZXIgb2JzZXJ2ZXMgdGhlIHRhc2sgYXMgc3VjY2VlZGVkXG4gICAgU3VjY2VlZGVkIC0tPiBbKl1cbiAgICBGYWlsZWQgLS0-IFsqXVxuIiwibWVybWFpZCI6e30sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :alt: The State diagram above illustrates the various states through which a Node transitions. This is the core FSM for a Node. + +The state diagram above illustrates the various states through which a Node transitions. This is the core FSM for a Node. +From a user's point of view, a Workflow simply consists of a sequence of tasks. But to Flyte, a Workflow internally creates a meta entity called a + +Once a Workflow enters a ``Running`` state, it triggers the phantom ``start node`` of the workflow. The Start node is always the entry node of any workflow. The start node starts executing all its child-nodes using a modified DepthFirst Search algorithm recursively. + +Nodes can be of different types, as follows, but all the nodes traverse through the same transitions + +#. Start Node - Only exists during the execution and is not modeled in the core spec +#. :std:ref:`gen/pb-protodoc/flyteidl/core/workflow.proto:flyteidl.core.tasknode` +#. :std:ref:`gen/pb-protodoc/flyteidl/core/workflow.proto:flyteidl.core.branchnode` +#. :std:ref:`gen/pb-protodoc/flyteidl/core/workflow.proto:flyteidl.core.workflownode` +#. Dynamic node - which is just a task node that does not return outputs, but futures. +#. End Node - only exists during the execution and is not modeled in the core spec + +Every transition between states is recorded in Flyteadmin using :std:ref:`gen/pb-protodoc/flyteidl/event/event.proto:flyteidl.event.nodeexecutionevent` + +Every NodeExecutionEvent can have one of the :std:ref:`api_enum_flyteidl.core.nodeexecution.phase` + +.. note:: TODO add explanation for each phase + +The state machine specification for the illustration can be found `here `_ + +Task States +================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBOb3RSZWFkeVxuICAgIFsqXSAtLT4gQWJvcnRlZCA6IEFib3J0ZWQgYnkgTm9kZUhhbmRsZXIgLSB0aW1lb3V0cywgZXh0cmVuYWwgYWJvcnQsIGV0Y1xuICAgIE5vdFJlYWR5IC0tPiBXYWl0aW5nRm9yUmVzb3VyY2VzIDogQmxvY2tlZCBvbiByZXNvdXJjZSBxdW90YSBvciByZXNvdXJjZSBwb29sIChvcHRpb25hbClcbiAgICBXYWl0aW5nRm9yUmVzb3VyY2VzIC0tPiBRdWV1ZWQgOiBIYXMgYmVlbiBzdWJtaXR0ZWQsIGJ1dCBoYXMgbm90IHN0YXJ0ZWQgKG9wdGlvbmFsKVxuICAgIFF1ZXVlZCAtLT4gSW5pdGlhbGl6aW5nIDogUHJlc3RhcnQgaW5pdGlhbGl6YXRpb24gKG9wdGlvbmFsKVxuICAgIEluaXRpYWxpemluZyAtLT4gUnVubmluZyA6IEFjdHVhbCBleGVjdXRpb24gb2YgdXNlciBjb2RlIGhhcyBzdGFydGVkXG4gICAgUnVubmluZyAtLT4gU3VjY2VzcyA6IFN1Y2Nlc3NmdWwgZXhlY3V0aW9uXG4gICAgUnVubmluZyAtLT4gUmV0cnlhYmxlRmFpbHVyZSA6IEZhaWxlZCB3aXRoIGEgcmV0cnlhYmxlIGVycm9yXG4gICAgUnVubmluZyAtLT4gUGVybWFuZW50RmFpbHVyZSA6IFVucmVjb3ZlcmFibGUgZmFpbHVyZSwgd2lsbCBzdG9wIGFsbCBleGVjdXRpb25cbiAgICBTdWNjZXNzIC0tPiBbKl1cbiAgICBSZXRyeWFibGVGYWlsdXJlIC0tPiBbKl1cbiAgICBQZXJtYW5lbnRGYWlsdXJlIC0tPiBbKl1cbiIsIm1lcm1haWQiOnt9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + :alt: The State diagram above illustrates the various states through which a Task transitions. This is the core FSM for any Task in Flyte. + +The State diagram above illustrates the various states through which a Task transitions. + +Every transition between states is recorded in Flyteadmin using :std:ref:`gen/pb-protodoc/flyteidl/event/event.proto:flyteidl.event.taskexecutionevent` + +Every TaskExecutionEvent can have one of the :std:ref:`api_enum_flyteidl.core.taskexecution.phase` + +.. note:: TODO add explanation for each phase + +The state machine specification for the illustration can be found `here `_ From 6db935ac4cd389228754ea5651b5682f5442d762 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 13 Apr 2021 13:58:54 -0700 Subject: [PATCH 52/92] Fix default eks storage config (#907) * Fix default eks storage config Signed-off-by: Haytham Abuelfutuh * kustomize Signed-off-by: Haytham Abuelfutuh * bump Signed-off-by: Haytham Abuelfutuh --- deployment/eks/flyte_generated.yaml | 54 +++++++++---------- .../eks/flyte/config/common/storage.yaml | 12 ++--- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index de4d19286b..a3407f511f 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -8080,14 +8080,12 @@ data: storage: type: stow stow: - kind: google + kind: s3 config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/devstorage.read_write - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" + auth_type: iam + region: us-east-2 + # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store + container: "flyte-demo" # NOTE this cache configuration is purely for propeller. But since we are having a common storage # config, we are configuring this value. In production create a separate storage config for # propeller and increase the cache size @@ -8098,7 +8096,7 @@ data: maxDownloadMBs: 10 kind: ConfigMap metadata: - name: datacatalog-config-2bb94tc5b5 + name: datacatalog-config-fktmhdgfk8 namespace: flyte --- apiVersion: v1 @@ -8178,14 +8176,12 @@ data: storage: type: stow stow: - kind: google + kind: s3 config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/devstorage.read_write - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" + auth_type: iam + region: us-east-2 + # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store + container: "flyte-demo" # NOTE this cache configuration is purely for propeller. But since we are having a common storage # config, we are configuring this value. In production create a separate storage config for # propeller and increase the cache size @@ -8207,7 +8203,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-t2f9g68k65 + name: flyte-admin-config-4ft85ftt4m namespace: flyte --- apiVersion: v1 @@ -8350,14 +8346,12 @@ data: storage: type: stow stow: - kind: google + kind: s3 config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/devstorage.read_write - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" + auth_type: iam + region: us-east-2 + # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store + container: "flyte-demo" # NOTE this cache configuration is purely for propeller. But since we are having a common storage # config, we are configuring this value. In production create a separate storage config for # propeller and increase the cache size @@ -8379,7 +8373,7 @@ data: cloudwatch-log-group: kind: ConfigMap metadata: - name: flyte-propeller-config-727k54546d + name: flyte-propeller-config-8fdc62895h namespace: flyte --- apiVersion: v1 @@ -8623,7 +8617,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: datacatalog-config-2bb94tc5b5 + name: datacatalog-config-fktmhdgfk8 name: config-volume - name: db-pass secret: @@ -8680,7 +8674,7 @@ spec: serviceAccountName: flyte-pod-webhook volumes: - configMap: - name: flyte-propeller-config-727k54546d + name: flyte-propeller-config-8fdc62895h name: config-volume - name: webhook-certs secret: @@ -8806,7 +8800,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-t2f9g68k65 + name: flyte-admin-config-4ft85ftt4m name: config-volume - configMap: name: clusterresource-template-tkdkkt4cb5 @@ -8898,7 +8892,7 @@ spec: serviceAccountName: flytepropeller volumes: - configMap: - name: flyte-propeller-config-727k54546d + name: flyte-propeller-config-8fdc62895h name: config-volume --- apiVersion: apps/v1 @@ -9086,7 +9080,7 @@ spec: name: clusterresource-template-tkdkkt4cb5 name: resource-templates - configMap: - name: flyte-admin-config-t2f9g68k65 + name: flyte-admin-config-4ft85ftt4m name: config-volume - name: db-pass secret: @@ -9129,7 +9123,7 @@ spec: serviceAccountName: flyte-pod-webhook volumes: - configMap: - name: flyte-propeller-config-727k54546d + name: flyte-propeller-config-8fdc62895h name: config-volume ttlSecondsAfterFinished: 0 --- diff --git a/kustomize/overlays/eks/flyte/config/common/storage.yaml b/kustomize/overlays/eks/flyte/config/common/storage.yaml index 6d678d0bbc..39dd393112 100644 --- a/kustomize/overlays/eks/flyte/config/common/storage.yaml +++ b/kustomize/overlays/eks/flyte/config/common/storage.yaml @@ -1,14 +1,12 @@ storage: type: stow stow: - kind: google + kind: s3 config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/devstorage.read_write - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" + auth_type: iam + region: us-east-2 + # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store + container: "flyte-demo" # NOTE this cache configuration is purely for propeller. But since we are having a common storage # config, we are configuring this value. In production create a separate storage config for # propeller and increase the cache size From 76e94e554459182163394bca957ab07538409b9c Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Thu, 15 Apr 2021 15:13:07 -0700 Subject: [PATCH 53/92] Remove flyteadmin config options which now have defaults (#917) Signed-off-by: Haytham Abuelfutuh --- deployment/eks/flyte_generated.yaml | 33 +++++-------------- deployment/gcp/flyte_generated.yaml | 29 +++++----------- deployment/sandbox/flyte_generated.yaml | 32 +++++------------- deployment/test/flyte_generated.yaml | 28 ++++------------ .../admindeployment/clustersync/cron.yaml | 2 +- .../base/admindeployment/deployment.yaml | 10 +++--- .../headless/config/admin/remote_data.yaml | 5 --- .../headless/config/admin/server.yaml | 8 ----- .../headless/kustomization.yaml | 1 - .../eks/flyte/config/admin/remote_data.yaml | 6 ---- .../overlays/eks/flyte/kustomization.yaml | 1 - kustomize/overlays/eks/kustomization.yaml | 4 +-- .../gcp/flyte/config/admin/remote_data.yaml | 2 -- .../overlays/gcp/flyte/kustomization.yaml | 1 - kustomize/overlays/gcp/kustomization.yaml | 4 +-- .../sandbox/flyte/admin/deployment.yaml | 6 ++-- kustomize/overlays/sandbox/kustomization.yaml | 2 +- .../overlays/test/flyte/admin/deployment.yaml | 6 ++-- kustomize/overlays/test/kustomization.yaml | 2 +- 19 files changed, 52 insertions(+), 130 deletions(-) delete mode 100644 kustomize/base/single_cluster/headless/config/admin/remote_data.yaml delete mode 100644 kustomize/overlays/eks/flyte/config/admin/remote_data.yaml delete mode 100644 kustomize/overlays/gcp/flyte/config/admin/remote_data.yaml diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index a3407f511f..46880630ce 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -8142,13 +8142,6 @@ data: logger: show-source: true level: 2 - remote_data.yaml: | - remoteData: - # TODO change this to match the region of the s3 bucket - region: "us-west-2" - scheme: aws - signedUrls: - durationMinutes: 3 server.yaml: | server: httpPort: 8088 @@ -8164,14 +8157,6 @@ data: - "Content-Type" flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin storage.yaml: | storage: type: stow @@ -8203,7 +8188,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-4ft85ftt4m + name: flyte-admin-config-ddg9df67hb namespace: flyte --- apiVersion: v1 @@ -8701,7 +8686,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.38 + app.kubernetes.io/version: 0.4.13 spec: containers: - command: @@ -8709,7 +8694,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8754,7 +8739,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8771,7 +8756,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8785,7 +8770,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8800,7 +8785,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-4ft85ftt4m + name: flyte-admin-config-ddg9df67hb name: config-volume - configMap: name: clusterresource-template-tkdkkt4cb5 @@ -9063,7 +9048,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -9080,7 +9065,7 @@ spec: name: clusterresource-template-tkdkkt4cb5 name: resource-templates - configMap: - name: flyte-admin-config-4ft85ftt4m + name: flyte-admin-config-ddg9df67hb name: config-volume - name: db-pass secret: diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml index 4e7ce1c0c5..5c4ae13030 100644 --- a/deployment/gcp/flyte_generated.yaml +++ b/deployment/gcp/flyte_generated.yaml @@ -8139,9 +8139,6 @@ data: logger: show-source: true level: 2 - remote_data.yaml: | - remoteData: - scheme: "gcp" server.yaml: | server: httpPort: 8088 @@ -8157,14 +8154,6 @@ data: - "Content-Type" flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin storage.yaml: | storage: type: stow @@ -8198,7 +8187,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-7k2dmcb4tg + name: flyte-admin-config-mhdft5gck2 namespace: flyte --- apiVersion: v1 @@ -8739,7 +8728,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.38 + app.kubernetes.io/version: 0.4.13 spec: containers: - command: @@ -8747,7 +8736,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8792,7 +8781,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8809,7 +8798,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8823,7 +8812,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8838,7 +8827,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-7k2dmcb4tg + name: flyte-admin-config-mhdft5gck2 name: config-volume - configMap: name: clusterresource-template-tkdkkt4cb5 @@ -9101,7 +9090,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -9118,7 +9107,7 @@ spec: name: clusterresource-template-tkdkkt4cb5 name: resource-templates - configMap: - name: flyte-admin-config-7k2dmcb4tg + name: flyte-admin-config-mhdft5gck2 name: config-volume - name: db-pass secret: diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index a6276ebb7b..54177e1b00 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -2152,12 +2152,6 @@ data: logger: show-source: true level: 2 - remote_data.yaml: | - remoteData: - region: "us-east-1" - scheme: "local" - signedUrls: - durationMinutes: 3 server.yaml: | server: httpPort: 8088 @@ -2173,14 +2167,6 @@ data: - "Content-Type" flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin storage.yaml: |+ storage: type: minio @@ -2206,7 +2192,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-5b5g7785h8 + name: flyte-admin-config-8kt2hhhb92 namespace: flyte --- apiVersion: v1 @@ -2843,7 +2829,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.38 + app.kubernetes.io/version: 0.4.13 spec: containers: - command: @@ -2851,7 +2837,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -2902,7 +2888,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -2918,7 +2904,7 @@ spec: - seed-projects - flytesnacks - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -2932,7 +2918,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -2950,7 +2936,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-5b5g7785h8 + name: flyte-admin-config-8kt2hhhb92 name: config-volume - name: db-pass secret: @@ -3337,7 +3323,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -3354,7 +3340,7 @@ spec: name: clusterresource-template-dtg8ff28mt name: resource-templates - configMap: - name: flyte-admin-config-5b5g7785h8 + name: flyte-admin-config-8kt2hhhb92 name: config-volume - name: db-pass secret: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 62205adc45..2a71096a1e 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -300,12 +300,6 @@ data: logger: show-source: true level: 2 - remote_data.yaml: | - remoteData: - region: "us-east-1" - scheme: "local" - signedUrls: - durationMinutes: 3 server.yaml: | server: httpPort: 8088 @@ -321,14 +315,6 @@ data: - "Content-Type" flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin storage.yaml: |+ storage: type: minio @@ -354,7 +340,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-k7f4k7g76c + name: flyte-admin-config-2mg752ch75 namespace: flyte --- apiVersion: v1 @@ -697,7 +683,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.38 + app.kubernetes.io/version: 0.4.13 spec: containers: - command: @@ -705,7 +691,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -756,7 +742,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -772,7 +758,7 @@ spec: - seed-projects - flytetester - flytesnacks - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -786,7 +772,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.7 + image: ghcr.io/flyteorg/flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -804,7 +790,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-k7f4k7g76c + name: flyte-admin-config-2mg752ch75 name: config-volume - name: db-pass secret: diff --git a/kustomize/base/admindeployment/clustersync/cron.yaml b/kustomize/base/admindeployment/clustersync/cron.yaml index b23ffaf6d5..e3fbfdfa69 100644 --- a/kustomize/base/admindeployment/clustersync/cron.yaml +++ b/kustomize/base/admindeployment/clustersync/cron.yaml @@ -12,7 +12,7 @@ spec: serviceAccountName: flyteadmin containers: - name: sync-cluster-resources - image: flyteadmin:v0.3.29 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml index b77ea096a1..3fcf8822ab 100644 --- a/kustomize/base/admindeployment/deployment.yaml +++ b/kustomize/base/admindeployment/deployment.yaml @@ -16,7 +16,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.38 + app.kubernetes.io/version: 0.4.13 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -37,7 +37,7 @@ spec: secretName: db-pass initContainers: - name: run-migrations - image: flyteadmin:v0.4.0 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "migrate", "run"] volumeMounts: @@ -47,7 +47,7 @@ spec: mountPath: /etc/db # Optional, These just seed the project - TODO move them to only - name: seed-projects - image: flyteadmin:v0.4.0 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"] @@ -57,7 +57,7 @@ spec: - name: db-pass mountPath: /etc/db - name: sync-cluster-resources - image: flyteadmin:v0.4.0 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] volumeMounts: @@ -69,7 +69,7 @@ spec: mountPath: /etc/db containers: - name: flyteadmin - image: flyteadmin:v0.4.0 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "serve"] ports: diff --git a/kustomize/base/single_cluster/headless/config/admin/remote_data.yaml b/kustomize/base/single_cluster/headless/config/admin/remote_data.yaml deleted file mode 100644 index 41719a0e66..0000000000 --- a/kustomize/base/single_cluster/headless/config/admin/remote_data.yaml +++ /dev/null @@ -1,5 +0,0 @@ -remoteData: - region: "us-east-1" - scheme: "local" - signedUrls: - durationMinutes: 3 diff --git a/kustomize/base/single_cluster/headless/config/admin/server.yaml b/kustomize/base/single_cluster/headless/config/admin/server.yaml index 253bf8c776..25a8ea4a56 100644 --- a/kustomize/base/single_cluster/headless/config/admin/server.yaml +++ b/kustomize/base/single_cluster/headless/config/admin/server.yaml @@ -12,11 +12,3 @@ server: - "Content-Type" flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin diff --git a/kustomize/base/single_cluster/headless/kustomization.yaml b/kustomize/base/single_cluster/headless/kustomization.yaml index c63c457102..d4cf4a7520 100644 --- a/kustomize/base/single_cluster/headless/kustomization.yaml +++ b/kustomize/base/single_cluster/headless/kustomization.yaml @@ -19,7 +19,6 @@ configMapGenerator: - ./config/admin/domain.yaml - ./config/admin/db.yaml - ./config/admin/cluster_resources.yaml - - ./config/admin/remote_data.yaml - ./config/admin/task_resource_defaults.yaml - ./config/common/storage.yaml - ./config/common/logger.yaml diff --git a/kustomize/overlays/eks/flyte/config/admin/remote_data.yaml b/kustomize/overlays/eks/flyte/config/admin/remote_data.yaml deleted file mode 100644 index 5e56e0740e..0000000000 --- a/kustomize/overlays/eks/flyte/config/admin/remote_data.yaml +++ /dev/null @@ -1,6 +0,0 @@ -remoteData: - # TODO change this to match the region of the s3 bucket - region: "us-west-2" - scheme: aws - signedUrls: - durationMinutes: 3 diff --git a/kustomize/overlays/eks/flyte/kustomization.yaml b/kustomize/overlays/eks/flyte/kustomization.yaml index 0da9ffcee5..ebc66e3425 100644 --- a/kustomize/overlays/eks/flyte/kustomization.yaml +++ b/kustomize/overlays/eks/flyte/kustomization.yaml @@ -27,7 +27,6 @@ configMapGenerator: - behavior: merge files: - ./config/admin/db.yaml - - ./config/admin/remote_data.yaml - ./config/admin/task_resource_defaults.yaml - ./config/common/storage.yaml name: flyte-admin-config diff --git a/kustomize/overlays/eks/kustomization.yaml b/kustomize/overlays/eks/kustomization.yaml index 4457715cff..4f64744c19 100644 --- a/kustomize/overlays/eks/kustomization.yaml +++ b/kustomize/overlays/eks/kustomization.yaml @@ -21,7 +21,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.7 # FLYTEADMIN_TAG override the tag + newTag: v0.4.13 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -38,4 +38,4 @@ images: # Webhook - name: webhook # match images with this name newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag - newName: ghcr.io/flyteorg/flytepropeller # override the name \ No newline at end of file + newName: ghcr.io/flyteorg/flytepropeller # override the name diff --git a/kustomize/overlays/gcp/flyte/config/admin/remote_data.yaml b/kustomize/overlays/gcp/flyte/config/admin/remote_data.yaml deleted file mode 100644 index 8af409c9a4..0000000000 --- a/kustomize/overlays/gcp/flyte/config/admin/remote_data.yaml +++ /dev/null @@ -1,2 +0,0 @@ -remoteData: - scheme: "gcp" diff --git a/kustomize/overlays/gcp/flyte/kustomization.yaml b/kustomize/overlays/gcp/flyte/kustomization.yaml index b369942368..bb08dcf05a 100644 --- a/kustomize/overlays/gcp/flyte/kustomization.yaml +++ b/kustomize/overlays/gcp/flyte/kustomization.yaml @@ -24,7 +24,6 @@ configMapGenerator: - behavior: merge files: - ./config/admin/db.yaml - - ./config/admin/remote_data.yaml - ./config/admin/task_resource_defaults.yaml - ./config/common/storage.yaml name: flyte-admin-config diff --git a/kustomize/overlays/gcp/kustomization.yaml b/kustomize/overlays/gcp/kustomization.yaml index 7fcca6183f..eeb26bfd26 100644 --- a/kustomize/overlays/gcp/kustomization.yaml +++ b/kustomize/overlays/gcp/kustomization.yaml @@ -23,7 +23,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.7 # FLYTEADMIN_TAG override the tag + newTag: v0.4.13 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -40,4 +40,4 @@ images: # Webhook - name: webhook # match images with this name newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag - newName: ghcr.io/flyteorg/flytepropeller # override the name \ No newline at end of file + newName: ghcr.io/flyteorg/flytepropeller # override the name diff --git a/kustomize/overlays/sandbox/flyte/admin/deployment.yaml b/kustomize/overlays/sandbox/flyte/admin/deployment.yaml index f3a7c1ef20..5a30adeb68 100644 --- a/kustomize/overlays/sandbox/flyte/admin/deployment.yaml +++ b/kustomize/overlays/sandbox/flyte/admin/deployment.yaml @@ -20,7 +20,7 @@ spec: do echo waiting for database; sleep 2; done;", ] - name: run-migrations - image: flyteadmin:v0.4.0 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: [ @@ -34,7 +34,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: flyteadmin:v0.4.0 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: [ @@ -50,7 +50,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: flyteadmin:v0.4.0 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: [ diff --git a/kustomize/overlays/sandbox/kustomization.yaml b/kustomize/overlays/sandbox/kustomization.yaml index 76c85d0d30..cfdb553c66 100644 --- a/kustomize/overlays/sandbox/kustomization.yaml +++ b/kustomize/overlays/sandbox/kustomization.yaml @@ -25,7 +25,7 @@ resources: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.7 # FLYTEADMIN_TAG override the tag + newTag: v0.4.13 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name diff --git a/kustomize/overlays/test/flyte/admin/deployment.yaml b/kustomize/overlays/test/flyte/admin/deployment.yaml index b4b9d59183..70db6afa6c 100644 --- a/kustomize/overlays/test/flyte/admin/deployment.yaml +++ b/kustomize/overlays/test/flyte/admin/deployment.yaml @@ -17,7 +17,7 @@ spec: 'until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done;'] - name: run-migrations - image: flyteadmin:v0.3.5 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "migrate", "run"] @@ -25,7 +25,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: flyteadmin:v0.3.5 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "migrate", "seed-projects", "flytetester", "flytesnacks"] @@ -33,7 +33,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: flyteadmin:v0.3.5 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/overlays/test/kustomization.yaml b/kustomize/overlays/test/kustomization.yaml index ddf689f58c..730d6af56c 100644 --- a/kustomize/overlays/test/kustomization.yaml +++ b/kustomize/overlays/test/kustomization.yaml @@ -19,7 +19,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.7 # FLYTEADMIN_TAG override the tag + newTag: v0.4.13 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name From 64e6e0ea43b3bc013c9150bb0e3688699fe3d8ef Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Tue, 20 Apr 2021 20:41:07 +0530 Subject: [PATCH 54/92] Contribution Guide (#922) Signed-off-by: Haytham Abuelfutuh --- CONTRIBUTING.md | 3 + rsts/community/contribute.rst | 255 ++++++++++++++++++++++++++++++++++ rsts/community/docs.rst | 144 ------------------- rsts/index.rst | 2 +- 4 files changed, 259 insertions(+), 145 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 rsts/community/contribute.rst delete mode 100644 rsts/community/docs.rst diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..f774b84d05 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,3 @@ +# Contributing to Flyte + +For information related to contributing to Flyte, please check out the [Contributing to Flyte](https://docs.flyte.org/en/latest/community/contribute.html) section of the documentation at the Flyte site. diff --git a/rsts/community/contribute.rst b/rsts/community/contribute.rst new file mode 100644 index 0000000000..bf1212adc1 --- /dev/null +++ b/rsts/community/contribute.rst @@ -0,0 +1,255 @@ +###################### +Contributing to Flyte +###################### + +Thank you for taking the time to contribute to Flyte! Here are some guidelines for you to follow, which will make your first and follow-up contributions easier. + +.. note:: + Please read our `Code of Conduct `_ before contributing to Flyte. + +Code +==== +An issue tagged with ``good first issue`` is the best place to start for first-time contributors. Look into them `here `_. + +To take a step ahead, check out the repositories available under `flyteorg `_. + +**Appetizer (for every repo): Fork and clone the concerned repository. Create a new branch on your fork and make the required changes. Create a pull request once your work is ready for review.** + +.. note:: + Note: To open a pull request, follow this `guide `_. + +*A piece of good news -- You can be added as a committer to any ``flyteorg`` repo as you become more involved with the project.* + +Example PR for your reference: `GitHub PR `_. A couple of checks are introduced to help in maintaining the robustness of the project. + +#. To get through DCO, sign off on every commit. (`Reference `_) +#. To improve code coverage, write unit tests to test your code. + +.. note:: + Format your Go code with ``golangci-lint`` followed by ``goimports`` (we used the same in the `Makefile `_), and Python code with ``black`` (use ``make fmt`` command which contains both black and isort). + +Environment Setup +***************** + +.. figure:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/contribution_guide/dependency_graph.png + :alt: Dependency Graph between various flyteorg repos + :align: center + :figclass: align-center + + The dependency graph between various flyteorg repos + +#. `flyte `_ + | Purpose: Deployment, Documentation, and Issues + | Languages: Kustomize & RST +#. `flyteidl `_ + | Purpose: The Flyte Workflow specification in `protocol buffers `_ which forms the core of Flyte + | Language: Protobuf + | Setup: Refer to the `README `_ +#. `flytepropeller `_ + | Purpose: Kubernetes native execution engine for Flyte Workflows and Tasks + | Language: Go + + Setup: + * Check for the Makefile in the root repo + * Run the following commands: + * ``make generate`` + * ``make test_unit`` + * ``make link`` + * To compile, run ``make compile`` +#. `flyteadmin `_ + | Purpose: Control Plane + | Language: Go + + Setup: + * Check for the Makefile in the root repo + * If the service code has to be tested, run it locally: + * ``make compile`` + * ``make server`` + * To seed data locally: + * ``make compile`` + * ``make seed_projects`` + * ``make migrate`` + * To run integration tests locally: + * ``make integration`` + * (or, to run in containerized dockernetes): ``make k8s_integration`` +#. `flytekit `_ + | Purpose: Python SDK & Tools + | Language: Python + | Setup: Refer to the `Flytekit Contribution Guide `_ +#. `flyteconsole `_ + | Purpose: Admin Console + | Language: Typescript + | Setup: Refer to the `README `_ +#. `datacatalog `_ + | Purpose: Manage Input & Output Artifacts + | Language: Go +#. `flyteplugins `_ + | Purpose: Flyte Plugins + | Language: Go + + Setup: + * Check for the Makefile in the root repo + * Run the following commands: + * ``make generate`` + * ``make test_unit`` + * ``make link`` +#. `flytestdlib `_ + | Purpose: Standard Library for Shared Components + | Language: Go +#. `flytesnacks `_ + | Purpose: Examples, Tips, and Tricks to use Flytekit SDKs + | Language: Python (In future, Java shall be added) + + Setup: + * If the Python code has to be tested, run it locally + * If the Python code has to be tested in a cluster: + * Run the ``make start`` command in the root directory of the flytesnacks repo + * Visit https://localhost:30081 to view the Flyte console consisting of the examples present in ``flytesnacks/cookbook/core`` directory + * To fetch the new dependencies and rebuild the image, run ``make register`` +#. `flytectl `_ + | Purpose: A Standalone Flyte CLI + | Language: Go + + Setup: + * Check for the Makefile in the root repo + * Run the following commands: + * ``make generate`` + * ``make test_unit`` + * ``make link`` + +Issues +====== +`GitHub Issues `_ is used for issue tracking. There are a variety of issue types available that you could use while filing an issue. + +* `Plugin Request `_ +* `Bug Report `_ +* `Documentation Bug/Update Request `_ +* `Core Feature Request `_ +* `Flytectl Feature Request `_ +* `Housekeeping `_ +* `UI Feature Request `_ + +If none of the above fits your requirements, file a `blank `_ issue. + +Documentation +============= +Flyte uses Sphinx for documentation and ``godocs`` for Golang. ``godocs`` is quite simple -- comment your code and you are good to go! + +Sphinx spans across multiple repositories under the `flyteorg `_ repository. It uses reStructured Text (rst) files to store the documentation content. For both the API and code-related content, it extracts docstrings from the code files. + +To get started, look into `reStructuredText reference `_. + +Environment Setup +***************** +Install all the requirements from the `docs-requirements.txt` file present in the root of a repository. + +.. code-block:: console + + pip install -r docs-requirements.txt + +From the ``docs`` directory present in the repository root (for ``flytesnacks``, ``docs`` is present in ``flytesnacks/cookbook``), run the command: + +.. code-block:: console + + make html + +.. note:: + For implicit targets, run ``make -C docs html``. + +You can then view the HTML pages in the ``docs/_build`` directory. + +.. note:: + For ``flyte`` repo, there is no ``docs`` directory. Instead, consider the ``rsts`` directory. To generate HTML files, run the following command in the root of the repo. + + .. code-block:: console + + make -C rsts html + +For minor edits that don’t require a local setup, you can edit the GitHub page in the documentation to propose the improvements. + +The edit option is found at the bottom of a page, as shown below. + +.. figure:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/contribution_guide/docs_edit.png + :alt: GitHub edit option for Documentation + :align: center + :figclass: align-center + +Intersphinx +*********** +`Intersphinx `_ can generate automatic links to the documentation of objects in other projects. + +To establish a reference to any other documentation from Flyte or within it, use intersphinx. + +To do so, create an ``intersphinx_mapping`` in the ``conf.py`` file present in the ``docs/source`` directory. + +For example: + +.. code-block:: python + + intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "flytekit": ("https://flyte.readthedocs.io/projects/flytekit/en/master/", None), + } + +.. note:: + ``docs/source`` is present in the repository root. Click `here `_ to view the intersphinx configuration. + +The key refers to the name used to refer to the file (while referencing the documentation), and the URL denotes the precise location. + +Here are a couple of examples that you can refer to: + +.. code-block:: text + + Task: :std:doc:`generated/flytekit.task` + +Output: + +Task: :std:doc:`generated/flytekit.task` + +.. code-block:: text + + :std:doc:`Using custom words ` + +Output: + +:std:doc:`Using custom words ` + +| + +Linking to Python elements changes based on what you're linking to. Check out this `section `_ to learn more. + +| + +For instance, linking to the `task` decorator in flytekit uses the ``func`` role. + +.. code-block:: text + + Link to flytekit code :py:func:`flytekit:flytekit.task` + +Output: + +Link to flytekit code :py:func:`flytekit:flytekit.task` + +| + +Here are a couple more examples. + +.. code-block:: text + + :py:mod:`Module ` + :py:class:`Class ` + :py:data:`Data ` + :py:func:`Function ` + :py:meth:`Method ` + +Output: + +:py:mod:`Module ` + +:py:class:`Class ` + +:py:data:`Data ` + +:py:func:`Function ` + +:py:meth:`Method ` \ No newline at end of file diff --git a/rsts/community/docs.rst b/rsts/community/docs.rst deleted file mode 100644 index 2e33df8440..0000000000 --- a/rsts/community/docs.rst +++ /dev/null @@ -1,144 +0,0 @@ -.. _contribute-docs: - -###################### -Contributing to Docs -###################### - -************************ -Docs for various repos -************************ -Flyte is a large project and all the docs span multiple repositories. The core of the documention is in the `flyteorg/flyte `_ repository. -Flyte uses `Sphinx `_ to compile it docs. Docs are automatically pushed on merge to master and docs are hosted using `readthedocs.org `_ - -*************** -Sphinx and RST -*************** - -Style -========= - -Headers --------- -Typically, we try to follow these characters in this order for heading separation. - -.. code-block:: text - - # with overline - * with overline - = - - - ^ - -Intersphinx -============= -`Intersphinx `__ is a plugin that all Flyte repos that build Sphinx documentation use for cross-referencing with each other. There's some good background information on it on these `slides `__. - -Inventory File ----------------- -When Sphinx runs, an inventory file gets created and is available alongside each repo's HTML pages. For example at ``https://readthedocs.org/projects/flytecookbook/objects.inv``. This file is a compressed inventory of all the sections, tags, etc. in the flyte cookbook documentation. This inventory file is what allows the intersphinx plugin to cross-link between projects. - -There is an open-source tool called ``sphobjinv`` that has managed to `reverse engineer these files `__, and offers a CLI to help search for things inside them. - -Setup -------- -Installing ``sphobjinv`` and simple usage :: - - $ pip install sphobjinv - - # Using the CLI to query a hosted inventory file, note the -u switch - $ sphobjinv suggest https://flytekit.readthedocs.io/en/latest/ -u task - - No inventory at provided URL. - Attempting "https://flytekit.readthedocs.io/en/latest/objects.inv" ... - Remote inventory found. - - :py:function:`flytekit.task` - :std:doc:`tasks` - :std:doc:`tasks.extend` - :std:label:`tasks:tasks` - - # Using the CLI to query a local file, useful when iterating locally - $ sphobjinv suggest ~/go/src/github.com/flyteorg/flytekit/docs/build/html/objects.inv task - - :py:function:`flytekit.task` - :std:doc:`tasks` - :std:doc:`tasks.extend` - :std:label:`tasks:tasks` - -.. note:: - - Even though the ``sphobjinv`` CLI returns ``:py:function:...``, when actually creating a link you should just use ``:py:func:...``. See `this `__. Here is a quick list of mappings - - .. list-table:: Conversion table for - ``sphobjinv`` - :widths: 50 50 - :header-rows: 1 - - * - What the tool returns? - - What you should use instead? - * - :py:module: - - :py:mod: - * - :py:function: - - :py:func: - * - :std:label: - - :std:ref: - * - :py:method: - - :py:meth: - - - -Linking Examples ------------------- -In the ``conf.py`` file of each repo, there is an intersphinx mapping argument that looks something like this :: - - intersphinx_mapping = { - "python": ("https://docs.python.org/3", None), - "flytekit": ("https://flyte.readthedocs.io/projects/flytekit/en/master/", None), - ... - } - -This file is what tells the plugin where to look for these inventory files, and what project name to refer to each inventory file as. The project name is important because they're used when actually referencing something from the inventory. - -Here are some examples, first the code and then the link - -.. code-block:: text - - Task: :std:doc:`generated/flytekit.task` - -Task: :std:doc:`generated/flytekit.task` - ------ - -.. code-block:: text - - :std:doc:`Using custom words ` - -:std:doc:`Using custom words ` - -Python -^^^^^^^ -Linking to Python elements changes based on what you're linking to. Check out this `section `__. For instance linking to the ``task`` decorator in flytekit uses the ``func`` role. - -.. code-block:: text - - Link to flytekit code :py:func:`flytekit:flytekit.task` - -Link to flytekit code :py:func:`flytekit:flytekit.task` - -Other elements use different Sphinx roles, here are some examples using Python core docs. :: - - :py:mod:`Module ` - :py:class:`Class ` - :py:data:`Data ` - :py:func:`Function ` - :py:meth:`Method ` - - -:py:mod:`Module ` - -:py:class:`Class ` - -:py:data:`Data ` - -:py:func:`Function ` - -:py:meth:`Method ` diff --git a/rsts/index.rst b/rsts/index.rst index 4fee64cdd9..b4b8bae45f 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -34,7 +34,7 @@ :hidden: Join the Community - community/docs + community/contribute community/roadmap community/compare From a30e70372e6b9a148f98cf7f91eee22c77c20d19 Mon Sep 17 00:00:00 2001 From: Jan Kumor Date: Thu, 22 Apr 2021 15:54:53 -0700 Subject: [PATCH 55/92] explicitly grant flyteworkflow finalizer permissions to flytepropeller (#930) * explicitly grant flyteworkflow finalizer permissions to flytepropeller Signed-off-by: akumor * add kustomize output after adding flyteworkflows/finalizers rbac Signed-off-by: akumor Signed-off-by: Haytham Abuelfutuh --- deployment/eks/flyte_generated.yaml | 1 + deployment/gcp/flyte_generated.yaml | 1 + deployment/sandbox/flyte_generated.yaml | 1 + deployment/test/flyte_generated.yaml | 1 + kustomize/base/propeller/rbac.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index 46880630ce..e8aeb32fa5 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -7737,6 +7737,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml index 5c4ae13030..0443de2487 100644 --- a/deployment/gcp/flyte_generated.yaml +++ b/deployment/gcp/flyte_generated.yaml @@ -7733,6 +7733,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 54177e1b00..55bbc31b83 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -1900,6 +1900,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 2a71096a1e..b6148e5cb0 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -134,6 +134,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list diff --git a/kustomize/base/propeller/rbac.yaml b/kustomize/base/propeller/rbac.yaml index 524740005e..951a48cc02 100644 --- a/kustomize/base/propeller/rbac.yaml +++ b/kustomize/base/propeller/rbac.yaml @@ -54,6 +54,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list From 8420bc47360e1177c87d2685ce00e79eddd61efd Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Thu, 22 Apr 2021 16:07:09 -0700 Subject: [PATCH 56/92] Add dynamic running state to node state machine (#929) Signed-off-by: Haytham Abuelfutuh --- rsts/dive_deep/state_machine.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rsts/dive_deep/state_machine.rst b/rsts/dive_deep/state_machine.rst index e04872db39..47457f4c15 100644 --- a/rsts/dive_deep/state_machine.rst +++ b/rsts/dive_deep/state_machine.rst @@ -36,7 +36,7 @@ The state machine specification for the illustration can be found `here `_ +The state machine specification for the illustration can be found `here `_ Task States ================ From e7c2893ca03a5d1f4a86ec089f1a9be2565ec6d1 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Wed, 28 Apr 2021 00:02:07 +0530 Subject: [PATCH 57/92] Updated Boilerplate Code (#932) Signed-off-by: Samhita Alla Signed-off-by: Haytham Abuelfutuh --- .github/config.yml | 15 ++++++++ Makefile | 6 +++ boilerplate/flyte/Readme.rst | 8 ++++ boilerplate/flyte/config.yml | 15 ++++++++ boilerplate/flyte/update.sh | 14 +++++++ boilerplate/update.cfg | 1 + boilerplate/update.sh | 75 ++++++++++++++++++++++++++++++++++++ 7 files changed, 134 insertions(+) create mode 100644 .github/config.yml create mode 100644 boilerplate/flyte/Readme.rst create mode 100644 boilerplate/flyte/config.yml create mode 100755 boilerplate/flyte/update.sh create mode 100644 boilerplate/update.cfg create mode 100755 boilerplate/update.sh diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 0000000000..f99bcd78f1 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,15 @@ +# Comment to be posted on PRs from first-time contributors in your repository +newPRWelcomeComment: > + Thank you for opening this pull request! 🙌 + + These tips will help get your PR across the finish line: + - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. + - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). + +# Comment to be posted to on pull requests merged by a first time user +firstPRMergeComment: > + Congrats on merging your first pull request! 🎉 + +# Comment to be posted on first-time issues +newIssueWelcomeComment: > + Thank you for opening your first issue here! 🛠 diff --git a/Makefile b/Makefile index 19cb24ee95..f343018352 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,13 @@ +export REPOSITORY=flyteidl + define PIP_COMPILE pip-compile $(1) --upgrade --verbose endef +.PHONY: update_boilerplate +update_boilerplate: + @boilerplate/update.sh + .PHONY: kustomize kustomize: KUSTOMIZE_VERSION=3.9.2 bash script/generate_kustomize.sh diff --git a/boilerplate/flyte/Readme.rst b/boilerplate/flyte/Readme.rst new file mode 100644 index 0000000000..ea18781185 --- /dev/null +++ b/boilerplate/flyte/Readme.rst @@ -0,0 +1,8 @@ +Config File -- Welcome Bot +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Provides a ``config.yml`` file. + +**To Enable:** + +Add ``flyte/config.yml`` to your ``boilerplate/update.cfg`` file. \ No newline at end of file diff --git a/boilerplate/flyte/config.yml b/boilerplate/flyte/config.yml new file mode 100644 index 0000000000..f99bcd78f1 --- /dev/null +++ b/boilerplate/flyte/config.yml @@ -0,0 +1,15 @@ +# Comment to be posted on PRs from first-time contributors in your repository +newPRWelcomeComment: > + Thank you for opening this pull request! 🙌 + + These tips will help get your PR across the finish line: + - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. + - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). + +# Comment to be posted to on pull requests merged by a first time user +firstPRMergeComment: > + Congrats on merging your first pull request! 🎉 + +# Comment to be posted on first-time issues +newIssueWelcomeComment: > + Thank you for opening your first issue here! 🛠 diff --git a/boilerplate/flyte/update.sh b/boilerplate/flyte/update.sh new file mode 100755 index 0000000000..506c3221b7 --- /dev/null +++ b/boilerplate/flyte/update.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +# Clone the config.yml file +echo " - copying ${DIR}/config.yml to the root directory." +cp ${DIR}/config.yml ${DIR}/../../.github/config.yml diff --git a/boilerplate/update.cfg b/boilerplate/update.cfg new file mode 100644 index 0000000000..90e8b6daac --- /dev/null +++ b/boilerplate/update.cfg @@ -0,0 +1 @@ +flyte/ diff --git a/boilerplate/update.sh b/boilerplate/update.sh new file mode 100755 index 0000000000..374e4b930e --- /dev/null +++ b/boilerplate/update.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +OUT="$(mktemp -d)" +trap "rm -fr $OUT" EXIT + +git clone git@github.com:flyteorg/boilerplate.git "${OUT}" + +echo "Updating the update.sh script." +cp "${OUT}/boilerplate/update.sh" "${DIR}/update.sh" +echo "" + + +CONFIG_FILE="${DIR}/update.cfg" +README="https://github.com/flyteorg/boilerplate/blob/master/Readme.rst" + +if [ ! -f "$CONFIG_FILE" ]; then + echo "$CONFIG_FILE not found." + echo "This file is required in order to select which features to include." + echo "See $README for more details." + exit 1 +fi + +if [ -z "$REPOSITORY" ]; then + echo '$REPOSITORY is required to run this script' + echo "See $README for more details." + exit 1 +fi + +while read directory junk; do + # Skip comment lines (which can have leading whitespace) + if [[ "$directory" == '#'* ]]; then + continue + fi + # Skip blank or whitespace-only lines + if [[ "$directory" == "" ]]; then + continue + fi + # Lines like + # valid/path other_junk + # are not acceptable, unless `other_junk` is a comment + if [[ "$junk" != "" ]] && [[ "$junk" != '#'* ]]; then + echo "Invalid config! Only one directory is allowed per line. Found '$junk'" + exit 1 + fi + + dir_path="${OUT}/boilerplate/${directory}" + # Make sure the directory exists + if ! [[ -d "$dir_path" ]]; then + echo "Invalid boilerplate directory: '$directory'" + exit 1 + fi + + echo "***********************************************************************************" + echo "$directory is configured in update.cfg." + echo "-----------------------------------------------------------------------------------" + echo "syncing files from source." + rm -rf "${DIR}/${directory}" + mkdir -p $(dirname "${DIR}/${directory}") + cp -r "$dir_path" "${DIR}/${directory}" + if [ -f "${DIR}/${directory}/update.sh" ]; then + echo "executing ${DIR}/${directory}/update.sh" + "${DIR}/${directory}/update.sh" + fi + echo "***********************************************************************************" + echo "" +done < "$CONFIG_FILE" \ No newline at end of file From 89ca2e7d0d176276cd7dfd77d97a2b2e3ae0725e Mon Sep 17 00:00:00 2001 From: Yuvraj <10830562+evalsocket@users.noreply.github.com> Date: Thu, 29 Apr 2021 00:34:48 +0530 Subject: [PATCH 58/92] Added goreleaser for publishing menifest (#927) * added goreleaser for publishing menifest Signed-off-by: yuvraj * Rename release file name Signed-off-by: yuvraj * Added script for artifacts Signed-off-by: yuvraj * More changes Signed-off-by: yuvraj * More changes Signed-off-by: yuvraj * fix typo Signed-off-by: yuvraj * minor changes Signed-off-by: yuvraj Signed-off-by: Haytham Abuelfutuh --- .github/workflows/publish-menifest.yml | 40 ++++++++++++++++++++++++++ .gitignore | 1 + .goreleaser.yml | 14 +++++++++ Makefile | 4 +++ script/prepare_artifacts.sh | 11 +++++++ 5 files changed, 70 insertions(+) create mode 100644 .github/workflows/publish-menifest.yml create mode 100644 .goreleaser.yml create mode 100644 script/prepare_artifacts.sh diff --git a/.github/workflows/publish-menifest.yml b/.github/workflows/publish-menifest.yml new file mode 100644 index 0000000000..834e3f51d6 --- /dev/null +++ b/.github/workflows/publish-menifest.yml @@ -0,0 +1,40 @@ +name: Create a flyte release +on: + workflow_dispatch: + inputs: + version: + description: 'version name. example v0.1.1' + required: true + +jobs: + goreleaser: + name: Goreleaser + runs-on: ubuntu-latest + outputs: + version: ${{ steps.bump-version.outputs.tag }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Bump version and push tag + id: bump-version + uses: anothrNick/github-tag-action@1.17.2 + env: + GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} + WITH_V: true + CUSTOM_TAG: ${{ github.event.inputs.version }} + RELEASE_BRANCHES: master + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + ref: ${{ github.event.inputs.version }} + - name: build Release Menifest + run: | + make prepare_artifacts + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9be7cc094f..5abe39323f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ docs/_sources/ docs/flytekit/flytekit.interfaces.html docs/searchindex.js docs/ +release/ __pycache__/ /helm/charts/ Chart.lock diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000000..a69fd22b81 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,14 @@ +project_name: flyte +builds: + - skip: true +release: + github: + owner: flyteorg + name: flyte + draft: false + prerelease: auto + discussion_category_name: General + name_template: "{{.ProjectName}}-v{{.Version}}" + disable: false + extra_files: + - glob: ./release/* \ No newline at end of file diff --git a/Makefile b/Makefile index f343018352..f88733e9e5 100644 --- a/Makefile +++ b/Makefile @@ -47,3 +47,7 @@ stats: @generate-dashboard -o deployment/stats/prometheus/flytepropeller-dashboard.json stats/flytepropeller_dashboard.py @generate-dashboard -o deployment/stats/prometheus/flyteadmin-dashboard.json stats/flyteadmin_dashboard.py @generate-dashboard -o deployment/stats/prometheus/flyteuser-dashboard.json stats/flyteuser_dashboard.py + +.PHONY: prepare_artifacts +prepare_artifacts: + bash script/prepare_artifacts.sh \ No newline at end of file diff --git a/script/prepare_artifacts.sh b/script/prepare_artifacts.sh new file mode 100644 index 0000000000..e2d60e7f8d --- /dev/null +++ b/script/prepare_artifacts.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -ex + +for file in ./deployment/**/flyte_generated.yaml; do + if [ -f "$file" ]; then + result=${file/#"./deployment/"} + result=${result/%"/flyte_generated.yaml"} + cp $file "./release/flyte_${result}_manifest.yaml" + fi +done From 0248dc901a3dca03486a106273cb25bf37478373 Mon Sep 17 00:00:00 2001 From: Yuvraj <10830562+evalsocket@users.noreply.github.com> Date: Thu, 29 Apr 2021 00:38:45 +0530 Subject: [PATCH 59/92] Added flytectl in sandbox (#952) * Added sandbox changes for new release process Signed-off-by: yuvraj * curl added in sandbox build image Signed-off-by: yuvraj Signed-off-by: Haytham Abuelfutuh --- docker/sandbox/Dockerfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/sandbox/Dockerfile b/docker/sandbox/Dockerfile index 0f70df8eaf..3d64453538 100644 --- a/docker/sandbox/Dockerfile +++ b/docker/sandbox/Dockerfile @@ -14,6 +14,9 @@ RUN git clone -b ${BUILDKIT_CLI_FOR_KUBECTL_VERSION} --single-branch --depth 1 h FROM alpine:3.13.2 AS base_ +# Install dependencies +RUN apk add --no-cache curl + # Make directory to store artifacts RUN mkdir -p /flyteorg/bin /flyteorg/share @@ -25,13 +28,16 @@ RUN wget -q -O /flyteorg/bin/k3s https://github.com/k3s-io/k3s/releases/download # Install buildkit-cli-for-kubectl COPY --from=go_builder_ /install/linux/ /flyteorg/bin/ -# Download flyte manifest +# Install flytectl +RUN curl https://raw.githubusercontent.com/flyteorg/flytectl/master/install.sh | sh + +# Copy flyte manifest COPY deployment/sandbox/flyte_generated.yaml /flyteorg/share/flyte_generated.yaml # Copy scripts +RUN cp /bin/flytectl /flyteorg/bin/ COPY docker/sandbox/kubectl docker/sandbox/wait-for-flyte.sh /flyteorg/bin/ - FROM base_ AS default # Install dependencies From ca95973dd74cefae34460da84ef4049b8f71916f Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 29 Apr 2021 23:58:43 +0530 Subject: [PATCH 60/92] Modify repo name in Makefile (#953) * Updated Boilerplate Code Signed-off-by: Samhita Alla * modified export line Signed-off-by: Samhita Alla Signed-off-by: Haytham Abuelfutuh --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f88733e9e5..6b38fd26ff 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -export REPOSITORY=flyteidl +export REPOSITORY=flyte define PIP_COMPILE pip-compile $(1) --upgrade --verbose From c168ec26030e5c03bfe2d4d340d9f0368fa18cbf Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Thu, 29 Apr 2021 20:59:26 -0400 Subject: [PATCH 61/92] Signed-off-by: Samhita Alla aallasamhita@gmail.com (#955) Co-authored-by: Niels Bantilan niels.bantilan@gmail.com Documentation revamp restructuring according to the RFC: https://docs.google.com/document/d/1Yp5cYujKT6UMv17Y6r1djMchaMZWAlPNUTZnbw1sZvc/edit#heading=h.vozt1qaes3ur Signed-off-by: cosmicBboy Co-authored-by: Samhita Alla Signed-off-by: Haytham Abuelfutuh --- doc-requirements.in | 2 +- doc-requirements.txt | 10 +- rsts/_static/custom.css | 91 ------------- rsts/community/compare.rst | 9 -- rsts/{dive_deep => concepts}/admin.rst | 0 .../{dive_deep => concepts}/admin_service.rst | 0 rsts/{dive_deep => concepts}/architecture.rst | 0 rsts/concepts/basics.rst | 18 +++ rsts/{dive_deep => concepts}/catalog.rst | 0 rsts/{dive_deep => concepts}/console.rst | 0 rsts/concepts/control_plane.rst | 13 ++ rsts/concepts/core.rst | 15 ++ .../customizable_resources.rst | 0 rsts/concepts/deployment_options.rst | 7 + rsts/{dive_deep => concepts}/domains.rst | 0 rsts/{dive_deep => concepts}/dynamic_spec.rst | 0 rsts/concepts/execution_time.rst | 14 ++ .../execution_timeline.rst | 0 rsts/{dive_deep => concepts}/executions.rst | 0 rsts/concepts/flyte_cli.rst | 7 + rsts/concepts/flyte_ui.rst | 7 + .../flyte_wf_tasks_high_level.png | Bin rsts/concepts/glossary.rst | 12 ++ .../launchplans_schedules.rst | 0 .../{dive_deep => concepts}/observability.rst | 0 rsts/{dive_deep => concepts}/overview.rst | 0 rsts/{dive_deep => concepts}/projects.rst | 0 rsts/{dive_deep => concepts}/registration.rst | 0 .../{dive_deep => concepts}/state_machine.rst | 0 rsts/{dive_deep => concepts}/tasks.rst | 0 .../workflows_nodes.rst | 0 rsts/conf.py | 4 - rsts/dive_deep/index.rst | 48 ------- rsts/howto/enable_and_use_schedules.rst | 2 +- rsts/howto/execute_workflow.rst | 2 +- rsts/howto/fast_registration.rst | 2 +- rsts/howto/install_sdk.rst | 2 +- rsts/howto/interruptible.rst | 2 +- rsts/howto/new_project.rst | 4 +- rsts/howto/performance/index.rst | 2 +- rsts/howto/resource_manager/index.rst | 2 +- rsts/howto/resource_quota.rst | 2 +- rsts/howto/sandbox.rst | 2 +- rsts/index.rst | 47 ++++--- rsts/plugins/aws/athena.rst | 9 -- rsts/plugins/aws/index.rst | 14 -- rsts/plugins/aws/sagemaker.rst | 9 -- rsts/plugins/extend/flyte_backend.rst | 75 ---------- rsts/plugins/extend/flytekit_python.rst | 10 -- rsts/plugins/extend/intro.rst | 128 ------------------ rsts/plugins/hive.rst | 9 -- rsts/plugins/index.rst | 27 +--- rsts/plugins/kubeflow/index.rst | 13 -- rsts/plugins/kubeflow/pytorch_operator.rst | 9 -- rsts/plugins/kubeflow/tensorflow_operator.rst | 9 -- rsts/plugins/pandera.rst | 9 -- rsts/plugins/papermill.rst | 9 -- rsts/plugins/pod.rst | 9 -- rsts/plugins/spark_k8s.rst | 2 +- rsts/plugins/sqlite3.rst | 9 -- 60 files changed, 142 insertions(+), 534 deletions(-) delete mode 100644 rsts/_static/custom.css delete mode 100644 rsts/community/compare.rst rename rsts/{dive_deep => concepts}/admin.rst (100%) rename rsts/{dive_deep => concepts}/admin_service.rst (100%) rename rsts/{dive_deep => concepts}/architecture.rst (100%) create mode 100644 rsts/concepts/basics.rst rename rsts/{dive_deep => concepts}/catalog.rst (100%) rename rsts/{dive_deep => concepts}/console.rst (100%) create mode 100644 rsts/concepts/control_plane.rst create mode 100644 rsts/concepts/core.rst rename rsts/{dive_deep => concepts}/customizable_resources.rst (100%) create mode 100644 rsts/concepts/deployment_options.rst rename rsts/{dive_deep => concepts}/domains.rst (100%) rename rsts/{dive_deep => concepts}/dynamic_spec.rst (100%) create mode 100644 rsts/concepts/execution_time.rst rename rsts/{dive_deep => concepts}/execution_timeline.rst (100%) rename rsts/{dive_deep => concepts}/executions.rst (100%) create mode 100644 rsts/concepts/flyte_cli.rst create mode 100644 rsts/concepts/flyte_ui.rst rename rsts/{dive_deep => concepts}/flyte_wf_tasks_high_level.png (100%) create mode 100644 rsts/concepts/glossary.rst rename rsts/{dive_deep => concepts}/launchplans_schedules.rst (100%) rename rsts/{dive_deep => concepts}/observability.rst (100%) rename rsts/{dive_deep => concepts}/overview.rst (100%) rename rsts/{dive_deep => concepts}/projects.rst (100%) rename rsts/{dive_deep => concepts}/registration.rst (100%) rename rsts/{dive_deep => concepts}/state_machine.rst (100%) rename rsts/{dive_deep => concepts}/tasks.rst (100%) rename rsts/{dive_deep => concepts}/workflows_nodes.rst (100%) delete mode 100644 rsts/dive_deep/index.rst delete mode 100644 rsts/plugins/aws/athena.rst delete mode 100644 rsts/plugins/aws/index.rst delete mode 100644 rsts/plugins/aws/sagemaker.rst delete mode 100644 rsts/plugins/extend/flyte_backend.rst delete mode 100644 rsts/plugins/extend/flytekit_python.rst delete mode 100644 rsts/plugins/extend/intro.rst delete mode 100644 rsts/plugins/hive.rst delete mode 100644 rsts/plugins/kubeflow/index.rst delete mode 100644 rsts/plugins/kubeflow/pytorch_operator.rst delete mode 100644 rsts/plugins/kubeflow/tensorflow_operator.rst delete mode 100644 rsts/plugins/pandera.rst delete mode 100644 rsts/plugins/papermill.rst delete mode 100644 rsts/plugins/pod.rst delete mode 100644 rsts/plugins/sqlite3.rst diff --git a/doc-requirements.in b/doc-requirements.in index 79fbabb86a..6d288ac2e7 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -1,4 +1,4 @@ -furo +git+git://github.com/flyteorg/furo@main readthedocs-sphinx-search sphinx sphinx-prompt diff --git a/doc-requirements.txt b/doc-requirements.txt index a801a17fd3..cd18b68296 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -6,9 +6,9 @@ # alabaster==0.7.12 # via sphinx -astroid==2.5.2 +astroid==2.5.6 # via sphinx-autoapi -babel==2.9.0 +babel==2.9.1 # via sphinx beautifulsoup4==4.9.3 # via @@ -20,7 +20,7 @@ chardet==4.0.0 # via requests docutils==0.16 # via sphinx -furo==2021.3.20b30 +git+git://github.com/flyteorg/furo@main # via -r doc-requirements.in idna==2.10 # via requests @@ -59,7 +59,7 @@ snowballstemmer==2.1.0 # via sphinx soupsieve==2.2.1 # via beautifulsoup4 -sphinx-autoapi==1.7.0 +sphinx-autoapi==1.8.1 # via -r doc-requirements.in sphinx-code-include==1.1.1 # via -r doc-requirements.in @@ -71,7 +71,7 @@ sphinx-prompt==1.4.0 # via -r doc-requirements.in sphinx-tabs==2.1.0 # via -r doc-requirements.in -sphinx==3.5.3 +sphinx==3.5.4 # via # -r doc-requirements.in # furo diff --git a/rsts/_static/custom.css b/rsts/_static/custom.css deleted file mode 100644 index d9851b7d8f..0000000000 --- a/rsts/_static/custom.css +++ /dev/null @@ -1,91 +0,0 @@ -h1, h2, h3, h4, h5, h6 { - font-weight: bold; -} - -.sidebar-logo { - max-width: 30%; -} - - -.sidebar-tree .reference.external:after { - content: none; -} - -.sphx-glr-thumbcontainer { - background-color: transparent; - border: transparent; -} - -.sphx-glr-thumbcontainer:hover { - border: transparent; -} - -div.sphx-glr-download a { - color:white; - background-color: #9d68e4cf; - background-image: none; - border: 1px solid #9d68e4cf; -} - -div.sphx-glr-download a:hover { - background-color: #8b48e2cf; - box-shadow: none; -} - -div.sphx-glr-thumbcontainer a.headerlink { - display: none; -} - -div.sphx-glr-thumbcontainer:hover { - border-color: white; - box-shadow: none; -} - -.sphx-glr-script-out .highlight pre { - background-color: #f8f8f8; -} - -p.sphx-glr-script-out { - padding-top: 0em; -} - -.search__outer::-webkit-scrollbar-track { - border-radius: 0px; -} - -@media (prefers-color-scheme: dark) { - .search__outer { - background-color: #131416 !important; - border: 1px solid #131416 !important; - } - .search__outer__input { - background-color: #1a1c1e !important; - } - .search__result__single { - border-bottom: #303335 !important; - } - .outer_div_page_results:hover { - background-color: black; - } - .search__result__title, .rtd_ui_search_subtitle { - color: #9D68E4 !important; - border-bottom: 1px solid #9D68E4 !important; - } - .search__outer .search__result__title span, .search__outer .search__result__content span { - background-color: #9d68e454; - } - .search__result__subheading, .search__result__content { - color: #ffffffd9 !important; - } - .search__outer::-webkit-scrollbar-track { - background-color: #131416 !important; - } - .rtd__search__credits { - background-color: #1a1c1e !important; - border: 1px solid #1a1c1e !important; - color: #81868d !important; - } - .rtd__search__credits a, .search__error__box { - color: #9ca0a5 !important; - } - } diff --git a/rsts/community/compare.rst b/rsts/community/compare.rst deleted file mode 100644 index b91540c15f..0000000000 --- a/rsts/community/compare.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _community_compare: - -################################### -Compare Flyte to other products -################################### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/dive_deep/admin.rst b/rsts/concepts/admin.rst similarity index 100% rename from rsts/dive_deep/admin.rst rename to rsts/concepts/admin.rst diff --git a/rsts/dive_deep/admin_service.rst b/rsts/concepts/admin_service.rst similarity index 100% rename from rsts/dive_deep/admin_service.rst rename to rsts/concepts/admin_service.rst diff --git a/rsts/dive_deep/architecture.rst b/rsts/concepts/architecture.rst similarity index 100% rename from rsts/dive_deep/architecture.rst rename to rsts/concepts/architecture.rst diff --git a/rsts/concepts/basics.rst b/rsts/concepts/basics.rst new file mode 100644 index 0000000000..8f970b2a5c --- /dev/null +++ b/rsts/concepts/basics.rst @@ -0,0 +1,18 @@ +.. _basics: + +###### +Basics +###### + +.. NOTE:: + + Coming soon 🛠 + +.. toctree:: + :maxdepth: 1 + :name: Basics + + flyte_ui + flyte_cli + deployment_options + glossary diff --git a/rsts/dive_deep/catalog.rst b/rsts/concepts/catalog.rst similarity index 100% rename from rsts/dive_deep/catalog.rst rename to rsts/concepts/catalog.rst diff --git a/rsts/dive_deep/console.rst b/rsts/concepts/console.rst similarity index 100% rename from rsts/dive_deep/console.rst rename to rsts/concepts/console.rst diff --git a/rsts/concepts/control_plane.rst b/rsts/concepts/control_plane.rst new file mode 100644 index 0000000000..83ebec2d36 --- /dev/null +++ b/rsts/concepts/control_plane.rst @@ -0,0 +1,13 @@ +################ +Control Plane +################ + +.. toctree:: + :maxdepth: 1 + + projects + domains + admin + admin_service + registration + console diff --git a/rsts/concepts/core.rst b/rsts/concepts/core.rst new file mode 100644 index 0000000000..d6eb0fb9c8 --- /dev/null +++ b/rsts/concepts/core.rst @@ -0,0 +1,15 @@ +.. _divedeep: + +############################ +Core Concepts & Architecture +############################ + +.. toctree:: + :maxdepth: 1 + :name: Concepts & Architecture + + overview + tasks + workflows_nodes + launchplans_schedules + architecture diff --git a/rsts/dive_deep/customizable_resources.rst b/rsts/concepts/customizable_resources.rst similarity index 100% rename from rsts/dive_deep/customizable_resources.rst rename to rsts/concepts/customizable_resources.rst diff --git a/rsts/concepts/deployment_options.rst b/rsts/concepts/deployment_options.rst new file mode 100644 index 0000000000..0ae2e5b1d1 --- /dev/null +++ b/rsts/concepts/deployment_options.rst @@ -0,0 +1,7 @@ +################################### +Deployment options (Local & Remote) +################################### + +.. NOTE:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/domains.rst b/rsts/concepts/domains.rst similarity index 100% rename from rsts/dive_deep/domains.rst rename to rsts/concepts/domains.rst diff --git a/rsts/dive_deep/dynamic_spec.rst b/rsts/concepts/dynamic_spec.rst similarity index 100% rename from rsts/dive_deep/dynamic_spec.rst rename to rsts/concepts/dynamic_spec.rst diff --git a/rsts/concepts/execution_time.rst b/rsts/concepts/execution_time.rst new file mode 100644 index 0000000000..c19980680b --- /dev/null +++ b/rsts/concepts/execution_time.rst @@ -0,0 +1,14 @@ +###################### +Execution Time Details +###################### + +.. toctree:: + :maxdepth: 1 + + executions + state_machine + execution_timeline + observability + dynamic_spec + catalog + customizable_resources \ No newline at end of file diff --git a/rsts/dive_deep/execution_timeline.rst b/rsts/concepts/execution_timeline.rst similarity index 100% rename from rsts/dive_deep/execution_timeline.rst rename to rsts/concepts/execution_timeline.rst diff --git a/rsts/dive_deep/executions.rst b/rsts/concepts/executions.rst similarity index 100% rename from rsts/dive_deep/executions.rst rename to rsts/concepts/executions.rst diff --git a/rsts/concepts/flyte_cli.rst b/rsts/concepts/flyte_cli.rst new file mode 100644 index 0000000000..da251e189e --- /dev/null +++ b/rsts/concepts/flyte_cli.rst @@ -0,0 +1,7 @@ +############## +Flyte CLI +############## + +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/concepts/flyte_ui.rst b/rsts/concepts/flyte_ui.rst new file mode 100644 index 0000000000..e26edf382a --- /dev/null +++ b/rsts/concepts/flyte_ui.rst @@ -0,0 +1,7 @@ +################# +Flyte UI +################# + +.. NOTE:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/flyte_wf_tasks_high_level.png b/rsts/concepts/flyte_wf_tasks_high_level.png similarity index 100% rename from rsts/dive_deep/flyte_wf_tasks_high_level.png rename to rsts/concepts/flyte_wf_tasks_high_level.png diff --git a/rsts/concepts/glossary.rst b/rsts/concepts/glossary.rst new file mode 100644 index 0000000000..0be14da2ad --- /dev/null +++ b/rsts/concepts/glossary.rst @@ -0,0 +1,12 @@ +############ +Glossary +############ + +.. glossary:: + + Memoization + Memoization ensures that a method doesn't run for the same inputs more than once by keeping a record of the results for the given inputs. + +.. NOTE:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/launchplans_schedules.rst b/rsts/concepts/launchplans_schedules.rst similarity index 100% rename from rsts/dive_deep/launchplans_schedules.rst rename to rsts/concepts/launchplans_schedules.rst diff --git a/rsts/dive_deep/observability.rst b/rsts/concepts/observability.rst similarity index 100% rename from rsts/dive_deep/observability.rst rename to rsts/concepts/observability.rst diff --git a/rsts/dive_deep/overview.rst b/rsts/concepts/overview.rst similarity index 100% rename from rsts/dive_deep/overview.rst rename to rsts/concepts/overview.rst diff --git a/rsts/dive_deep/projects.rst b/rsts/concepts/projects.rst similarity index 100% rename from rsts/dive_deep/projects.rst rename to rsts/concepts/projects.rst diff --git a/rsts/dive_deep/registration.rst b/rsts/concepts/registration.rst similarity index 100% rename from rsts/dive_deep/registration.rst rename to rsts/concepts/registration.rst diff --git a/rsts/dive_deep/state_machine.rst b/rsts/concepts/state_machine.rst similarity index 100% rename from rsts/dive_deep/state_machine.rst rename to rsts/concepts/state_machine.rst diff --git a/rsts/dive_deep/tasks.rst b/rsts/concepts/tasks.rst similarity index 100% rename from rsts/dive_deep/tasks.rst rename to rsts/concepts/tasks.rst diff --git a/rsts/dive_deep/workflows_nodes.rst b/rsts/concepts/workflows_nodes.rst similarity index 100% rename from rsts/dive_deep/workflows_nodes.rst rename to rsts/concepts/workflows_nodes.rst diff --git a/rsts/conf.py b/rsts/conf.py index f778cbe4de..feaf81a969 100644 --- a/rsts/conf.py +++ b/rsts/conf.py @@ -111,10 +111,6 @@ pygments_style = "tango" pygments_dark_style = "native" -html_css_files = [ - "custom.css", -] - html_theme_options = { "light_css_variables": { "color-brand-primary": "#4300c9", diff --git a/rsts/dive_deep/index.rst b/rsts/dive_deep/index.rst deleted file mode 100644 index 9efea9094f..0000000000 --- a/rsts/dive_deep/index.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. _divedeep: - -########################################## -Architecture and Concepts -########################################## - -******************************** -Core Concepts & Architecture -******************************** - -.. toctree:: - :maxdepth: 1 - :name: Concepts & Architecture - - overview - tasks - workflows_nodes - launchplans_schedules - architecture - -******************************** -Control Plane Details -******************************** - -.. toctree:: - :maxdepth: 1 - - projects - domains - admin - admin_service - registration - console - -******************************** -Execution Time Details -******************************** - -.. toctree:: - :maxdepth: 1 - - executions - state_machine - execution_timeline - observability - dynamic_spec - catalog - customizable_resources diff --git a/rsts/howto/enable_and_use_schedules.rst b/rsts/howto/enable_and_use_schedules.rst index 6bea7a0b65..9bc719282f 100644 --- a/rsts/howto/enable_and_use_schedules.rst +++ b/rsts/howto/enable_and_use_schedules.rst @@ -165,7 +165,7 @@ Workflow Executor ----------------- Scheduled events which trigger need to be handled by the workflow executor, which subscribes to triggered events from the SQS queue you've configured above. -.. CAUTION:: +.. NOTE:: Failure to configure a workflow executor will result in all your scheduled events piling up silently without ever kicking off workflow executions. diff --git a/rsts/howto/execute_workflow.rst b/rsts/howto/execute_workflow.rst index eee7f9179d..3e4234831a 100644 --- a/rsts/howto/execute_workflow.rst +++ b/rsts/howto/execute_workflow.rst @@ -4,6 +4,6 @@ How do I execute a workflow? #################################### -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/fast_registration.rst b/rsts/howto/fast_registration.rst index cd3ef6e5e9..60b124dba4 100644 --- a/rsts/howto/fast_registration.rst +++ b/rsts/howto/fast_registration.rst @@ -4,7 +4,7 @@ How do I use Fast Registration? ******************************** -.. caution:: Experimental feature (beta) +.. NOTE:: Experimental feature (beta) Are you frustrated by having to wait for an image build in order to test out simple code changes to your Flyte workflows? If you're interested in reducing to your iteration cycle to mere seconds, read on below. diff --git a/rsts/howto/install_sdk.rst b/rsts/howto/install_sdk.rst index 111fcc1413..0d2cb45a18 100644 --- a/rsts/howto/install_sdk.rst +++ b/rsts/howto/install_sdk.rst @@ -21,6 +21,6 @@ All Flytekiplugins are also published to pypi as independent libraries and can b How to install Flytekit Java? ################################# -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/interruptible.rst b/rsts/howto/interruptible.rst index 934e75291a..1abcb5f8ea 100644 --- a/rsts/howto/interruptible.rst +++ b/rsts/howto/interruptible.rst @@ -49,6 +49,6 @@ Most Flyte workloads should be good candidates for spot instances. If your task How to recover from interruptions? =================================== -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/new_project.rst b/rsts/howto/new_project.rst index 78e766c0f3..9ccd0e32b2 100644 --- a/rsts/howto/new_project.rst +++ b/rsts/howto/new_project.rst @@ -7,7 +7,7 @@ How do I create/register a new project? Using flytectl --------------- -.. CAUTION:: +.. NOTE:: Coming soon 🛠 @@ -27,6 +27,6 @@ If you refresh your console you'll see your new project appear! FlyteAdmin API reference ------------------------- -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/performance/index.rst b/rsts/howto/performance/index.rst index 5851f4e059..9c862102e5 100644 --- a/rsts/howto/performance/index.rst +++ b/rsts/howto/performance/index.rst @@ -4,6 +4,6 @@ How do I optimize performance of my Flyte Deployment? ###################################################### -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/resource_manager/index.rst b/rsts/howto/resource_manager/index.rst index ba96a480f4..3ed54ad535 100644 --- a/rsts/howto/resource_manager/index.rst +++ b/rsts/howto/resource_manager/index.rst @@ -5,6 +5,6 @@ How do I enable and configure resource manager? ################################################# -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/resource_quota.rst b/rsts/howto/resource_quota.rst index 79f7d4c926..ac609818c2 100644 --- a/rsts/howto/resource_quota.rst +++ b/rsts/howto/resource_quota.rst @@ -4,6 +4,6 @@ How do I limit resources per project/domain? ############################################### -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/sandbox.rst b/rsts/howto/sandbox.rst index c0e3a1bd09..d8305fca72 100644 --- a/rsts/howto/sandbox.rst +++ b/rsts/howto/sandbox.rst @@ -154,4 +154,4 @@ Deploy Flyte Sandbox environment to a shared kubernetes cluster The goal here is to deploy to an existing Kubernetes cluster - within one namespace only. This would allow multiple Flyte clusters to run within one K8s cluster. -.. caution:: coming soon! +.. NOTE:: coming soon! diff --git a/rsts/index.rst b/rsts/index.rst index b4b8bae45f..bdd1765d67 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -3,41 +3,54 @@ .. toctree:: :maxdepth: 1 :name: mainsections + :titlesonly: :hidden: getting_started - Tutorials - reference/index + User Guide + Tutorials + Concepts + API Reference community/index .. toctree:: - :caption: How-Tos - :maxdepth: 1 - :name: howtotoc - :hidden: - - plugins/index - howto/index - -.. toctree:: - :caption: Deep Dive - :maxdepth: 1 + :caption: Concepts + :maxdepth: -1 :name: divedeeptoc :hidden: - dive_deep/index + concepts/basics + concepts/core + concepts/control_plane + concepts/execution_time .. toctree:: - :caption: Developers - :maxdepth: 1 + :caption: Community + :maxdepth: -1 :name: roadmaptoc :hidden: Join the Community community/contribute community/roadmap - community/compare +.. toctree:: + :caption: API Reference + :maxdepth: -1 + :name: apireference + :hidden: + + References + +.. toctree:: + :caption: How-Tos + :maxdepth: 1 + :name: howtotoc + :hidden: + + plugins/index + howto/index + Meet Flyte ========== diff --git a/rsts/plugins/aws/athena.rst b/rsts/plugins/aws/athena.rst deleted file mode 100644 index 8d5898b674..0000000000 --- a/rsts/plugins/aws/athena.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-aws-athena: - -############### -AWS Athena -############### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/aws/index.rst b/rsts/plugins/aws/index.rst deleted file mode 100644 index d9197ae8de..0000000000 --- a/rsts/plugins/aws/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _plugins-aws: - -############# -AWS Plugins -############# - -.. toctree:: - :maxdepth: 1 - :caption: Available Plugins - :name: pluginsawstoc - - athena - sagemaker - diff --git a/rsts/plugins/aws/sagemaker.rst b/rsts/plugins/aws/sagemaker.rst deleted file mode 100644 index 628f0d288a..0000000000 --- a/rsts/plugins/aws/sagemaker.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-aws-sagemaker: - -############### -AWS Sagemaker -############### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/extend/flyte_backend.rst b/rsts/plugins/extend/flyte_backend.rst deleted file mode 100644 index 8791fcf590..0000000000 --- a/rsts/plugins/extend/flyte_backend.rst +++ /dev/null @@ -1,75 +0,0 @@ -.. _extend-plugin-flyte-backend: - -######################################## -Implement Backend Extensions (advanced) -######################################## - -Now that you have landed here, we can assume that you have exhausted your options of extending and want to extend Flyte in a way that adds new capabilities to the platform. - -Let us try to recap, why we should implement a backend plugin, - -#. We want to add a new capability to the Flyte Platform, for e.g. - * ability to talk to a new service like AWS Sagemaker, Snowflake, Redshift, Athena, BigQuery etc - * ability to orchestrate a set of containers in a new way like Spark, Flink, Distributed training on Kubernetes (usually using a Kubernetes operator) - * use a new container orchestration engine like AWS Batch/ECS, Hashicorp' Nomad - * use a completely new runtime like AWS Lambda, KNative etc -#. For the case of talking to a new service like in 1.a - this can be done using flytekit extensions and usually is the better way to get started. But, once matured most of these extensions are better to be migrated to the backend. For the rest of the cases, it is possible to extend flytekit to achieve these scenarios, but this is less desirable, because of the associated overhead of first launching a container that launches these jobs downstream. -#. You want to retain the capability to updating the plugin implementation and roll out new changes, fixes without affecting the users code, or requiring them to update versions of their plugins etc -#. You want the same plugin to be accessible across multiple language SDK's - build a universal plugin - -Basics -======= -We will try to understand the components of a backend plugin using an example plugin - :ref:`plugins-spark-k8s` A Flyte backend extension consists of 3 parts - -Interface specification ------------------------- -Usually Flyte extensions need information that is not covered by a :std:ref:`Flyte TaskTemplate `. The TaskTemplate consists of a -the interface, task_type identifier, some metadata and other fields. An important field to note here is - :std:ref:`api_field_flyteidl.core.tasktemplate.custom`. The custom field is essentially an unstructured JSON. -This makes it possible to extend a task-template beyond the default supported targets -- :std:ref:`api_field_flyteidl.core.tasktemplate.container` (WIP, sql etc). - -The motivation of the Custom field, is to marshal a JSON structure that specifies information beyond what a regular TaskTemplate can capture. The actual structure of the JSON is known only to the implemented backend-plugin and the SDK components. The core Flyte platform, does not understand of look into the specifics of this structure. -It is highly recommended to use an interface definition lanugage like Protobuf, OpenAPISpec etc to declare specify the structure of the JSON. From here, on we refer to this as the ``Plugin Specification``. - -For Spark we decided to use Protobuf to specify the plugin as can be seen `here `__. Note it is not necessary to have the Plugin structure specified in FlyteIDL, we do it for simplicity, ease of maintenance alongwith the core platform and because of existing tooling to generate code for protobuf. - -Flytekit Plugin implementation --------------------------------- -Now that you have a specification, we have to implement a method to generate this new TaskTemplate, with the special custom field. Also, this is where the UX design comes into play. You want to write the best possible interface in the SDK that users are delighted to use. The end goal is to create the TaskTemplate with the Custom field populated with the actual JSON structure. -We will currently refer to Flytekit - python as an example for extending and implementing the SDK. (For java refer to other docs). -The SDK task should be implemented as an extension of :py:class:`flytekit.extend.PythonTask`, or more commonly :py:class:`flytekit.PythonFunctionTask`. -In the case of Spark, we extend the :py:class:`flytekit.PythonFunctionTask`, as shown `here `__. - -The SparkTask is implemented as a regular flytekit plugin, with one exception, the ``Custom`` field is now actually the ``SparkJob`` protocol buffer. Flytekit base classes when serializing a task, will automatically invoke the `get_custom method `_. - - -FlytePropeller backend Plugin ------------------------------- -The backend plugin is where the actual logic of the execution is implemented. The backend plugin uses Flyte - PluginMachinery inteface to implement a plugin which can be one of the following supported types - -#. A `Kubernetes operator Plugin `_ -#. A Web API plugin - `Async `_ or `Sync `_. -#. Or if none of the above fits then - a `Core Plugin `_ - -Kubernetes operator Plugin -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. CAUTION:: - - Coming soon 🛠 - - -Web API plugin -^^^^^^^^^^^^^^^ - -.. CAUTION:: - - Coming soon 🛠 - - -Catch-all - Core Plugin -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. CAUTION:: - - Coming soon 🛠 - \ No newline at end of file diff --git a/rsts/plugins/extend/flytekit_python.rst b/rsts/plugins/extend/flytekit_python.rst deleted file mode 100644 index 2bd1f1de4f..0000000000 --- a/rsts/plugins/extend/flytekit_python.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _extend-plugin-flytekit-python: - -################################## -Extend flytekit (python) -################################## - -Extending Flytekit is desirable whether you are writing a backend plugins or a Flytekit only plugin. In this section we will cover the basics of how to extend Flytekit to add new plugins - task-types. -In the next section of :ref:`extend-plugin-flyte-backend`, we will talk about backend plugins, which includes extending flytekit - but specifically for the backend plugin. - -.. caution:: Work in progress. For a simple example of how to write a flytekit plugin refer to :std:ref:`advanced_custom_task_plugin`. But, remember flytekit can be extended beyond this simple method - so feel free to ask us a question in the slack channel. \ No newline at end of file diff --git a/rsts/plugins/extend/intro.rst b/rsts/plugins/extend/intro.rst deleted file mode 100644 index 18b7896393..0000000000 --- a/rsts/plugins/extend/intro.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. _plugins_extend_intro: - -########################### -When & How to Extend Flyte -########################### - -.. caution:: These docs are still work in progress. Please read through and if you have any questions don't shy away from either filing a github issue or ping us in the Slack channel. The community loves plugins and would love to help you in any way. - -The Core of Flyte is a container execution engine, where you can write one or more tasks and string them together to form a data dependency DAG - called a ``workflow``. -If your work involves writing simple python or java tasks that can either perform operations on their own or can call out to external services - then there is **NO NEED to extend FLYTE**. - -But, in that case you can almost do everything using python / java or a container - So why should you even have to extend Flyte? - -================= -But First - Why? -================= - -Case 1: I want to use my special Types - e.g. my own DataFrame format -========================================================================== -Flyte, just like a programming language has a core type-system, but just like most languages, this type system can be extended by allowing users to add ``User defined Data types``. -A User defined data type can be something that Flyte does not really understand, but is extremely useful for a users specific needs. For example it can be a custom user structure or a grouping of images in a specific encoding. - -Flytekit natively supports handling of structured data like User defined structures like DataClasses using JSON as the representation format. An example of this is available in FlyteCookbook - :std:doc:`auto_core_intermediate/custom_objects`. - -For types that are not simply representable as JSON documents, Flytekit allows users to extends Flyte's type system and implement these types in Python. The user has to essentially implement a :py:class:`flytekit.extend.TypeTransformer` class to enable translation of the type from Users type to flyte understood types. As an example, -instead of using :py:class:`pandas.DataFrame` directly, you may want to use `Pandera `_ to perform validation of an input or output dataframe. an example can be found `here `_. - -To extend the type system in flytekit refer to an illustrative example found at - :std:ref:`advanced_custom_types`. - - -Case 2: Add a new Task Type - Flyte capability -=============================================== -So often times you want to interact with a service like, - - - a Database (Postgres, MySQL, etc) - - a DataWarehouse like (Snowflake, BigQuery, Redshift etc) - - a computation platform like (AWS EMR, Databricks etc) - -and you want this to be available like a template for all other users - open source or within your organization. This can be done by creating a task plugin. -A Task-plugin makes it possible for you or other users to use your idea natively within Flyte as this capability was built into the flyte platform. - -Thus for example, if you want users to write code simply using the ``@task`` decorator, but you want to provide a capability of running the function as a spark job or a sagemaker training job - then you can extend Flyte's task system - we will refer to this as the plugin and it could be possible to do the following - -.. code-block:: python - - @task(task_config=MyContainerExecutionTask( - plugin_specific_config_a=..., - plugin_specific_config_b=..., - ... - )) - def foo(...) -> ...: - ... - - -OR provide an interface like this - -.. code-block:: python - - query_task = SnowflakeQuery(query="Select * from x where x.time < {{.inputs.time}}", inputs=(time=datetime), results=pandas.DataFrame) - - @workflow - def my_wf(t: datetime) -> ...: - df = query_task(time=t) - return process(df=df) - - - -=========================================================== -I want to write a Task Plugin or add a new TaskType -=========================================================== - -Interestingly there are 2 options here. You can write a task plugin simply as an extension in flytekit, or you can go deeper and write a Plugin in the Flyte backend itself. - -Flytekit only plugin -====================== -An illustrative example of writing a flytekit plugin can be found at - :std:ref:`advanced_custom_task_plugin`. Flytekit plugins are simple to write and should invariably be -the first place you start at. Here - -**Pros** - -#. Simple to write, just implement in python. Flyte will treat it like a container execution and blindly pass control to the plugin -#. Simple to publish - flytekitplugins can be published as independent libraries and they follow a simple api. -#. Simple to perform testing - just test locally in flytekit - -**Cons** - -#. Limited ways of providing additional visibility in progress, or external links etc -#. Has to be implemented again in every language as these are SDK side plugins only -#. In case of side-effects, potentially of causing resource leaks. For example if the plugins runs a BigQuery Job, it is possible that the plugin may crash after running the Job and Flyte cannot guarantee that the BigQuery job wil be successfully terminated. -#. Potentially expensive - In cases where the plugin just runs a remote job - e.g how Airflow does, then running a new pod for every task execution causes severe strain on k8s and the task itself uses almost no CPUs. Also because of stateful natute, using spot-instances is not trivial. -#. A bug fix to the runtime, needs a new library version of the plugin -#. Not trivial to implement resource controls - e.g. throttling, resource pooling etc - -Backend Plugin -=============== - -Doc on how to writed a backend plugins is coming soon. A backend plugin essentially makes it possible for users to write extensions for FlytePropeller (Flytes scheduling engine). This enables complete control on the visualization and availability of the plugin. - -**Pros** - -#. Service oriented way of deploying new plugins - strong contracts. Maintainers can deploy new versions of the backend plugin, fix bugs, without needing the users to upgrade Libraries etc -#. Drastically cheaper and more efficient to execute. FlytePropeller is written in Golang and uses an event loop model. Each process of FlytePropeller can execute 1000's of tasks concurrently. -#. Flyte will guarantee resource cleanup -#. Flyteconsole plugins (capability coming soon) can be added to customize visualization and progress tracking of the execution -#. Resource controls and backpressure management is available -#. Implement once, use in any SDK or language - -**Cons** - -#. Need to be implemented in golang -#. Needs a FlytePropeller build - *currently* -#. Need to implement contract in some spec language like protobf, openAPI etc -#. Development cycle can be much slower than flytekit only plugins - - -=============================================== -How do I decide which path to take? -=============================================== - -.. image:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/core/extend_flyte_flowchart.png - :alt: Ok you want to add a plugin, but which type? Follow the flowchart and then select the right next steps. - - -Use the conclusion of the flow-chart to refer to the right doc -================================================================ - -- :ref:`extend-plugin-flytekit-python` -- :ref:`extend-plugin-flyte-backend` diff --git a/rsts/plugins/hive.rst b/rsts/plugins/hive.rst deleted file mode 100644 index 55e078c214..0000000000 --- a/rsts/plugins/hive.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-hive: - -########## -BYOC Hive -########## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/index.rst b/rsts/plugins/index.rst index e59f565fb7..d502c7a060 100644 --- a/rsts/plugins/index.rst +++ b/rsts/plugins/index.rst @@ -1,23 +1,5 @@ .. _plugins: -################ -Extending Flyte -################ - -.. _plugins_howto: - -Flyte as platform was designed with extensibility as a core primitive. Flyte is essentially an integration framework and hence extensibility is possible through-out the system. -The following sections will guide you through writing your own extensions - either private or public (contribute back to the community). - -.. toctree:: - :maxdepth: 1 - :name: howtoextendtoc - - extend/intro - extend/flytekit_python - extend/flyte_backend - - ==================== Available Extensions ==================== @@ -27,11 +9,4 @@ The following is a list of maintained plugins for Flyte and guides on how to ins :maxdepth: 1 :name: pluginstoc - spark_k8s - pod - sqlite3 - pandera - papermill - hive - aws/index - kubeflow/index + spark_k8s \ No newline at end of file diff --git a/rsts/plugins/kubeflow/index.rst b/rsts/plugins/kubeflow/index.rst deleted file mode 100644 index c770acda03..0000000000 --- a/rsts/plugins/kubeflow/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _plugins-kubeflow: - -################ -Kubeflow Plugins -################ - -.. toctree:: - :maxdepth: 1 - :caption: Available Plugins - :name: pluginskftoc - - tensorflow_operator - pytorch_operator \ No newline at end of file diff --git a/rsts/plugins/kubeflow/pytorch_operator.rst b/rsts/plugins/kubeflow/pytorch_operator.rst deleted file mode 100644 index 6beb4c1226..0000000000 --- a/rsts/plugins/kubeflow/pytorch_operator.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-kf-pytorch-operator: - -################# -Pytorch Operator -################# - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/kubeflow/tensorflow_operator.rst b/rsts/plugins/kubeflow/tensorflow_operator.rst deleted file mode 100644 index a198557e5e..0000000000 --- a/rsts/plugins/kubeflow/tensorflow_operator.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-kf-tf-operator: - -############### -TF Operator -############### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/pandera.rst b/rsts/plugins/pandera.rst deleted file mode 100644 index 004219d100..0000000000 --- a/rsts/plugins/pandera.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-pandera: - -################################################## -Use Pandera to Enforce Type safety in DataFrames -################################################## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/papermill.rst b/rsts/plugins/papermill.rst deleted file mode 100644 index 7edd202a1d..0000000000 --- a/rsts/plugins/papermill.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-papermill: - -################################## -Use Papermill notebooks as tasks -################################## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/pod.rst b/rsts/plugins/pod.rst deleted file mode 100644 index 1aca0fee46..0000000000 --- a/rsts/plugins/pod.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-pod: - -######## -K8s Pods -######## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/spark_k8s.rst b/rsts/plugins/spark_k8s.rst index 0b4aa016fc..772da4e598 100644 --- a/rsts/plugins/spark_k8s.rst +++ b/rsts/plugins/spark_k8s.rst @@ -11,7 +11,7 @@ Flyte has an optional plugin that makes it possible to run `Apache Spark DataFrames -################################################## - -.. CAUTION:: - - Coming soon 🛠 From 13190b06c556ea37cef49d3e348db93d58851d06 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 30 Apr 2021 22:12:04 +0530 Subject: [PATCH 62/92] added tagline (#956) Signed-off-by: Samhita Alla Signed-off-by: Haytham Abuelfutuh --- rsts/index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rsts/index.rst b/rsts/index.rst index bdd1765d67..b3b0f6af9e 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -54,6 +54,11 @@ Meet Flyte ========== +.. raw:: html + +

The workflow automation platform for complex, mission-critical data and ML processes at scale

+ + Flyte is an open-source, container-native, structured programming and distributed processing platform. It enables highly concurrent, scalable and maintainable workflows for machine learning and data processing. Created at `Lyft `__ in collaboration with Spotify, Freenome and many others, Flyte provides first class support for Python, Java, and Scala, and is built directly on Kubernetes for all the benefits containerization provides: portability, scalability, and reliability. From bbbff7841ff40f167a0379da770c2085f8b84b44 Mon Sep 17 00:00:00 2001 From: Tom Forbes Date: Fri, 30 Apr 2021 17:42:43 +0100 Subject: [PATCH 63/92] Fix broken Kustomize link in docs (#944) Signed-off-by: Tom Forbes Signed-off-by: Haytham Abuelfutuh --- kustomize/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/README.md b/kustomize/README.md index 4189f9a146..94f0f6ec56 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -1,5 +1,5 @@ # Install Flyte using Kustomize -Flyte can be deployed to a kubernetes cluster using a generated deployment yaml file. This file is generated using [Kustomize](https://kubectl.docs.kubernetes.io/pages/reference/kustomize.html). +Flyte can be deployed to a kubernetes cluster using a generated deployment yaml file. This file is generated using [Kustomize](https://kubectl.docs.kubernetes.io/guides/introduction/kustomize/). Please refer to Kustomize documentation to see how it works. In brief, Kustomize allows composing a deployment yaml using multiple components. In Flyte all the core components are listed under [Base Components](./base). The Base components also consist of From 88a3174657765da1bd3ba9b83c390e1c299bd638 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 30 Apr 2021 16:50:25 -0400 Subject: [PATCH 64/92] update getting started guide (#958) since we're consolidating our docs, we want to use the getting started example originally from flytesnacks. Here we're moving it into the flyte repo since the flytesnacks example should be the canonical getting started guide Signed-off-by: cosmicBboy Signed-off-by: Haytham Abuelfutuh --- rsts/getting_started.rst | 99 ++++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 35 deletions(-) diff --git a/rsts/getting_started.rst b/rsts/getting_started.rst index 09b8e7c545..f21ddc9d7c 100644 --- a/rsts/getting_started.rst +++ b/rsts/getting_started.rst @@ -3,72 +3,101 @@ Getting started --------------- -.. rubric:: Estimated time: 3 minutes +.. rubric:: Estimated time to complete: 3 minutes. Prerequisites -############# +*************** Make sure you have `docker installed `__ and `git `__ installed, then install flytekit: +Steps +***** + +1. First install the python Flytekit SDK and clone the ``flytesnacks`` repo: + .. prompt:: bash - pip install flytekit + pip install --pre flytekit + git clone git@github.com:flyteorg/flytesnacks.git flytesnacks + cd flytesnacks -Clone the `flytekit-python-template `__ repo to create our own git repository called ``flyteexamples``: +2. The repo comes with some useful Make targets to make your experimentation workflow easier. Run ``make help`` to get the supported commands. + Let's start a sandbox cluster: .. prompt:: bash - git clone git@github.com:flyteorg/flytekit-python-template.git flyteexamples - cd flyteexamples + make start +3. Take a minute to explore Flyte Console through the provided URL. -Write Your First Flyte Workflow -############################### +.. image:: https://github.com/flyteorg/flyte/raw/static-resources/img/first-run-console-2.gif + :alt: A quick visual tour for launching your first Workflow. +4. Open ``hello_world.py`` in your favorite editor. -Let's take a look at the example workflow in `myapp/workflows/example.py `__: +.. code-block:: -.. rli:: https://raw.githubusercontent.com/flyteorg/flytekit-python-template/main/myapp/workflows/example.py - :language: python + cookbook/core/basic/hello_world.py -As you can see, a Flyte :std:doc:`task ` is the most basic unit of work in Flyte, -and you can compose multiple tasks into a :std:doc:`workflow `. Try running and -modifying the ``example.py`` script locally. +5. Add ``name: str`` as an argument to both ``my_wf`` and ``say_hello`` functions. Then update the body of ``say_hello`` to consume that argument. -Start a Local Flyte Backend -########################### +.. tip:: -Once you're happy with the ``example.py`` script, run the following command in your terminal: + .. code-block:: python -.. prompt:: bash + @task + def say_hello(name: str) -> str: + return f"hello world, {name}" + +.. tip:: + + .. code-block:: python + + @workflow + def my_wf(name: str) -> str: + res = say_hello(name=name) + return res + +6. Update the simple test at the bottom of the file to pass in a name. E.g. - docker run --rm --privileged -p 30081:30081 -p 30082:30082 -p 30084:30084 ghcr.io/flyteorg/flyte-sandbox +.. tip:: -When you see the message ``Flyte is ready!``, your local sandbox should be ready on http://localhost:30081/console. + .. code-block:: python -Register Your Workflows -########################### + print(f"Running my_wf(name='adam') {my_wf(name='adam')}") -Now we're ready to ship your code to the Flyte backend by running the following command: +7. When you run this file locally, it should output ``hello world, adam``. Run this command in your terminal: .. prompt:: bash - FLYTE_AWS_ENDPOINT=http://localhost:30084/ FLYTE_AWS_ACCESS_KEY_ID=minio FLYTE_AWS_SECRET_ACCESS_KEY=miniostorage make fast_register + python cookbook/core/basic/hello_world.py + +*Congratulations!* You have just run your first workflow. Now, let's run it on the sandbox cluster deployed earlier. + +8. Run: + +.. prompt:: bash + + REGISTRY=ghcr.io/flyteorg make fast_register + +9. Visit `the console `__, click launch, and enter your name as the input. -Run Your Workflows -################## +10. Give it a minute and once it's done, check out "Inputs/Outputs" on the top right corner to see your updated greeting. -To run a workflow, go to http://localhost:30081/console/projects/flyteexamples/workflows and then follow these steps: +.. image:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/flytesnacks/tutorial/exercise.gif + :alt: A quick visual tour for launching a workflow and checking the outputs when they're done. -1. Select the ``hello_world`` workflow -2. Click the **Launch Workflow** button in the upper right corner -3. Update the ``name`` input argument -4. Proceed to **Launch** to trigger an execution +.. admonition:: Recap -.. rubric:: 🎉 Congratulations, you just ran your first Flyte workflow 🎉 + You have successfully: + 1. Run a flyte sandbox cluster, + 2. Run a flyte workflow locally, + 3. Run a flyte workflow on a cluster. -Next Steps: Tutorials -##################### + .. rubric:: 🎉 Congratulations, you just ran your first Flyte workflow 🎉 -To experience the full capabilities of Flyte, try out the `Flytekit Tutorials `__ 🛫 + Next Steps: User Guide + ####################### + + To experience the full capabilities of Flyte, take a look at the `User Guide `__ 🛫 From f8a2a310f0d0071e4f304056e88a9cf13888c4ac Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 4 May 2021 17:20:43 -0700 Subject: [PATCH 65/92] Authentication Config Examples (#970) * Fix broken Kustomize link in docs (#944) Signed-off-by: Tom Forbes Signed-off-by: Haytham Abuelfutuh * update getting started guide (#958) since we're consolidating our docs, we want to use the getting started example originally from flytesnacks. Here we're moving it into the flyte repo since the flytesnacks example should be the canonical getting started guide Signed-off-by: cosmicBboy Signed-off-by: Haytham Abuelfutuh * Authentication Config Examples Signed-off-by: Haytham Abuelfutuh * Feedback Signed-off-by: Haytham Abuelfutuh * fix map specification in cluster resource template Signed-off-by: Haytham Abuelfutuh * attempting stdlib fix Signed-off-by: Haytham Abuelfutuh * Fix merge issue Signed-off-by: Haytham Abuelfutuh * Update Signed-off-by: Haytham Abuelfutuh * Spellcheck Signed-off-by: Haytham Abuelfutuh * regenerate Signed-off-by: Haytham Abuelfutuh * Update migration.rst Signed-off-by: Haytham Abuelfutuh * Update setup.rst Signed-off-by: Haytham Abuelfutuh Co-authored-by: Tom Forbes Co-authored-by: Niels Bantilan Co-authored-by: SandraGH5 <80421934+SandraGH5@users.noreply.github.com> Signed-off-by: Haytham Abuelfutuh --- deployment/eks/flyte_generated.yaml | 127 +++++++-- deployment/gcp/flyte_generated.yaml | 127 +++++++-- deployment/sandbox/flyte_generated.yaml | 127 +++++++-- deployment/test/flyte_generated.yaml | 123 +++++++-- .../base/admindeployment/auth_secret.yaml | 6 + .../base/admindeployment/deployment.yaml | 195 ++++++++------ .../base/admindeployment/kustomization.yaml | 5 +- kustomize/base/propeller/auth_secret.yaml | 8 + kustomize/base/propeller/deployment.yaml | 47 ++-- kustomize/base/propeller/kustomization.yaml | 1 + .../config/admin/cluster_resources.yaml | 30 +-- .../headless/config/admin/server.yaml | 29 ++ kustomize/overlays/eks/kustomization.yaml | 8 +- kustomize/overlays/gcp/kustomization.yaml | 8 +- kustomize/overlays/sandbox/kustomization.yaml | 8 +- kustomize/overlays/test/kustomization.yaml | 8 +- rsts/howto/authentication/flyte_cli_flow.png | Bin 41554 -> 0 bytes rsts/howto/authentication/flyte_ui_flow.png | Bin 47268 -> 0 bytes rsts/howto/authentication/index.rst | 131 +++++---- rsts/howto/authentication/migration.rst | 151 +++++++++++ rsts/howto/authentication/setup.rst | 252 ++++++++++++++++++ 21 files changed, 1093 insertions(+), 298 deletions(-) create mode 100644 kustomize/base/admindeployment/auth_secret.yaml create mode 100644 kustomize/base/propeller/auth_secret.yaml delete mode 100644 rsts/howto/authentication/flyte_cli_flow.png delete mode 100644 rsts/howto/authentication/flyte_ui_flow.png create mode 100644 rsts/howto/authentication/migration.rst create mode 100644 rsts/howto/authentication/setup.rst diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index e8aeb32fa5..24dbf837d2 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -8106,21 +8106,21 @@ data: cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" customData: - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "3000Mi" refresh: 5m db.yaml: | database: @@ -8148,7 +8148,9 @@ data: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -8156,6 +8158,33 @@ data: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" storage.yaml: | @@ -8189,7 +8218,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-ddg9df67hb + name: flyte-admin-config-dck98m94b6 namespace: flyte --- apiVersion: v1 @@ -8372,6 +8401,22 @@ metadata: type: Opaque --- apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +stringData: + client_secret: foobar +type: Opaque +--- +apiVersion: v1 kind: Service metadata: annotations: @@ -8567,7 +8612,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/flyteorg/datacatalog:v0.3.1 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -8590,7 +8635,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/datacatalog:v0.3.1 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8647,7 +8692,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -8695,7 +8740,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8713,6 +8758,8 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - mountPath: /etc/secrets/ + name: auth - command: - sh - -c @@ -8740,7 +8787,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8757,7 +8804,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8771,7 +8818,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8781,12 +8828,28 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - args: + - flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth + command: + - /bin/sh + - -c + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + imagePullPolicy: IfNotPresent + name: generate-secrets + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume serviceAccountName: flyteadmin volumes: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-ddg9df67hb + name: flyte-admin-config-dck98m94b6 name: config-volume - configMap: name: clusterresource-template-tkdkkt4cb5 @@ -8794,6 +8857,9 @@ spec: - name: db-pass secret: secretName: db-pass-bthd2588cc + - name: auth + secret: + secretName: flyte-admin-auth --- apiVersion: apps/v1 kind: Deployment @@ -8862,7 +8928,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -8875,11 +8941,16 @@ spec: volumeMounts: - mountPath: /etc/flyte/config name: config-volume + - mountPath: /etc/secrets/ + name: auth serviceAccountName: flytepropeller volumes: - configMap: name: flyte-propeller-config-8fdc62895h name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth --- apiVersion: apps/v1 kind: Deployment @@ -9049,7 +9120,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -9066,7 +9137,7 @@ spec: name: clusterresource-template-tkdkkt4cb5 name: resource-templates - configMap: - name: flyte-admin-config-ddg9df67hb + name: flyte-admin-config-dck98m94b6 name: config-volume - name: db-pass secret: @@ -9099,7 +9170,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml index 0443de2487..58f77c77f8 100644 --- a/deployment/gcp/flyte_generated.yaml +++ b/deployment/gcp/flyte_generated.yaml @@ -8104,21 +8104,21 @@ data: cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" customData: - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "3000Mi" refresh: 5m db.yaml: | database: @@ -8145,7 +8145,9 @@ data: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -8153,6 +8155,33 @@ data: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" storage.yaml: | @@ -8188,7 +8217,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-mhdft5gck2 + name: flyte-admin-config-864hmfdc2m namespace: flyte --- apiVersion: v1 @@ -8369,6 +8398,22 @@ metadata: type: Opaque --- apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +stringData: + client_secret: foobar +type: Opaque +--- +apiVersion: v1 kind: Service metadata: name: cloudsqlproxy @@ -8609,7 +8654,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/flyteorg/datacatalog:v0.3.1 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -8632,7 +8677,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/datacatalog:v0.3.1 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8689,7 +8734,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -8737,7 +8782,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8755,6 +8800,8 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - mountPath: /etc/secrets/ + name: auth - command: - sh - -c @@ -8782,7 +8829,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8799,7 +8846,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8813,7 +8860,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8823,12 +8870,28 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - args: + - flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth + command: + - /bin/sh + - -c + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + imagePullPolicy: IfNotPresent + name: generate-secrets + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume serviceAccountName: flyteadmin volumes: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-mhdft5gck2 + name: flyte-admin-config-864hmfdc2m name: config-volume - configMap: name: clusterresource-template-tkdkkt4cb5 @@ -8836,6 +8899,9 @@ spec: - name: db-pass secret: secretName: db-pass-bthd2588cc + - name: auth + secret: + secretName: flyte-admin-auth --- apiVersion: apps/v1 kind: Deployment @@ -8904,7 +8970,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -8917,11 +8983,16 @@ spec: volumeMounts: - mountPath: /etc/flyte/config name: config-volume + - mountPath: /etc/secrets/ + name: auth serviceAccountName: flytepropeller volumes: - configMap: name: flyte-propeller-config-k8km2mmdgf name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth --- apiVersion: apps/v1 kind: Deployment @@ -9091,7 +9162,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -9108,7 +9179,7 @@ spec: name: clusterresource-template-tkdkkt4cb5 name: resource-templates - configMap: - name: flyte-admin-config-mhdft5gck2 + name: flyte-admin-config-864hmfdc2m name: config-volume - name: db-pass secret: @@ -9141,7 +9212,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 55bbc31b83..fda4d1b493 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -2118,21 +2118,21 @@ data: cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" customData: - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "3000Mi" refresh: 5m db.yaml: | database: @@ -2158,7 +2158,9 @@ data: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -2166,6 +2168,33 @@ data: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" storage.yaml: |+ @@ -2193,7 +2222,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-8kt2hhhb92 + name: flyte-admin-config-7gmg5ffhdd namespace: flyte --- apiVersion: v1 @@ -2454,6 +2483,22 @@ metadata: type: Opaque --- apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +stringData: + client_secret: foobar +type: Opaque +--- +apiVersion: v1 data: password: bXl1c2Vy user_secret: bXlzZWNyZXQ= @@ -2709,7 +2754,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/flyteorg/datacatalog:v0.3.1 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -2727,7 +2772,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/datacatalog:v0.3.1 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -2784,7 +2829,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -2838,7 +2883,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -2856,6 +2901,8 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - mountPath: /etc/secrets/ + name: auth - command: - sh - -c @@ -2889,7 +2936,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -2905,7 +2952,7 @@ spec: - seed-projects - flytesnacks - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -2919,7 +2966,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -2929,6 +2976,22 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - args: + - flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth + command: + - /bin/sh + - -c + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + imagePullPolicy: IfNotPresent + name: generate-secrets + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume serviceAccountName: flyteadmin volumes: - configMap: @@ -2937,11 +3000,14 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-8kt2hhhb92 + name: flyte-admin-config-7gmg5ffhdd name: config-volume - name: db-pass secret: secretName: db-pass-9dgchhk2bm + - name: auth + secret: + secretName: flyte-admin-auth --- apiVersion: apps/v1 kind: Deployment @@ -3010,7 +3076,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -3018,11 +3084,16 @@ spec: volumeMounts: - mountPath: /etc/flyte/config name: config-volume + - mountPath: /etc/secrets/ + name: auth serviceAccountName: flytepropeller volumes: - configMap: name: flyte-propeller-config-6gd7cgkkdt name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth --- apiVersion: apps/v1 kind: Deployment @@ -3324,7 +3395,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -3341,7 +3412,7 @@ spec: name: clusterresource-template-dtg8ff28mt name: resource-templates - configMap: - name: flyte-admin-config-8kt2hhhb92 + name: flyte-admin-config-7gmg5ffhdd name: config-volume - name: db-pass secret: @@ -3508,7 +3579,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index b6148e5cb0..536197903f 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -266,21 +266,21 @@ data: cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" customData: - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "3000Mi" refresh: 5m db.yaml: | database: @@ -306,7 +306,9 @@ data: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -314,6 +316,33 @@ data: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" storage.yaml: |+ @@ -341,7 +370,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-2mg752ch75 + name: flyte-admin-config-g5mt4b25k8 namespace: flyte --- apiVersion: v1 @@ -458,6 +487,22 @@ metadata: type: Opaque --- apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +stringData: + client_secret: foobar +type: Opaque +--- +apiVersion: v1 kind: Service metadata: annotations: @@ -569,7 +614,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/flyteorg/datacatalog:v0.3.1 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -587,7 +632,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/datacatalog:v0.3.1 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -644,7 +689,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: webhook volumeMounts: @@ -692,7 +737,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -710,6 +755,8 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - mountPath: /etc/secrets/ + name: auth - command: - sh - -c @@ -743,7 +790,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -759,7 +806,7 @@ spec: - seed-projects - flytetester - flytesnacks - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -773,7 +820,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.4.13 + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -783,6 +830,22 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - args: + - flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth + command: + - /bin/sh + - -c + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + imagePullPolicy: IfNotPresent + name: generate-secrets + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume serviceAccountName: flyteadmin volumes: - configMap: @@ -791,11 +854,14 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-2mg752ch75 + name: flyte-admin-config-g5mt4b25k8 name: config-volume - name: db-pass secret: secretName: db-pass-9dgchhk2bm + - name: auth + secret: + secretName: flyte-admin-auth --- apiVersion: apps/v1 kind: Deployment @@ -830,7 +896,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -838,11 +904,16 @@ spec: volumeMounts: - mountPath: /etc/flyte/config name: config-volume + - mountPath: /etc/secrets/ + name: auth serviceAccountName: flytepropeller volumes: - configMap: name: flyte-propeller-config-dtdt5gbgg2 name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth --- apiVersion: apps/v1 kind: Deployment @@ -944,7 +1015,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flytepropeller:v0.10.0 + image: ghcr.io/flyteorg/flytepropeller:v0.10.7 imagePullPolicy: IfNotPresent name: webhook volumeMounts: diff --git a/kustomize/base/admindeployment/auth_secret.yaml b/kustomize/base/admindeployment/auth_secret.yaml new file mode 100644 index 0000000000..9a1281d776 --- /dev/null +++ b/kustomize/base/admindeployment/auth_secret.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml index 3fcf8822ab..e574dfaa39 100644 --- a/kustomize/base/admindeployment/deployment.yaml +++ b/kustomize/base/admindeployment/deployment.yaml @@ -24,81 +24,126 @@ spec: spec: serviceAccountName: flyteadmin volumes: - - name: shared-data - emptyDir: {} - - name: config-volume - configMap: - name: flyte-admin-config - - name: resource-templates - configMap: - name: clusterresource-template - - name: db-pass - secret: - secretName: db-pass - initContainers: - - name: run-migrations - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "migrate", "run"] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - # Optional, These just seed the project - TODO move them to only - - name: seed-projects - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", - "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - - name: sync-cluster-resources - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] - volumeMounts: - - name: resource-templates - mountPath: /etc/flyte/clusterresource/templates - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - containers: - - name: flyteadmin - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "serve"] - ports: - - containerPort: 8088 - - containerPort: 8089 - volumeMounts: - name: shared-data - mountPath: /srv/flyte + emptyDir: {} - name: config-volume - mountPath: /etc/flyte/config + configMap: + name: flyte-admin-config + - name: resource-templates + configMap: + name: clusterresource-template - name: db-pass - mountPath: /etc/db - - name: redoc - image: docker.io/redocly/redoc - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8087 - resources: - limits: - memory: "200Mi" - cpu: "0.1" - command: - - sh - - -c - - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh - env: - - name: PAGE_TITLE - value: "Flyte Admin OpenAPI" - - name: SPEC_URL - value: "/api/v1/openapi" - - name: PORT - value: "8087" + secret: + secretName: db-pass + - name: auth + secret: + secretName: flyte-admin-auth + initContainers: + - name: run-migrations + image: flyteadmin:v0.4.13 + imagePullPolicy: IfNotPresent + command: + [ + "flyteadmin", + "--config", + "/etc/flyte/config/*.yaml", + "migrate", + "run", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: db-pass + mountPath: /etc/db + # Optional, These just seed the project - TODO move them to only + - name: seed-projects + image: flyteadmin:v0.4.13 + imagePullPolicy: IfNotPresent + command: + [ + "flyteadmin", + "--config", + "/etc/flyte/config/*.yaml", + "migrate", + "seed-projects", + "flytesnacks", + "flytetester", + "flyteexamples", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: db-pass + mountPath: /etc/db + - name: sync-cluster-resources + image: flyteadmin:v0.4.13 + imagePullPolicy: IfNotPresent + command: + [ + "flyteadmin", + "--config", + "/etc/flyte/config/*.yaml", + "clusterresource", + "sync", + ] + volumeMounts: + - name: resource-templates + mountPath: /etc/flyte/clusterresource/templates + - name: config-volume + mountPath: /etc/flyte/config + - name: db-pass + mountPath: /etc/db + - name: generate-secrets + image: flyteadmin:v0.4.0 + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + [ + "flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + containers: + - name: flyteadmin + image: flyteadmin:v0.4.13 + imagePullPolicy: IfNotPresent + command: + ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "serve"] + ports: + - containerPort: 8088 + - containerPort: 8089 + volumeMounts: + - name: shared-data + mountPath: /srv/flyte + - name: config-volume + mountPath: /etc/flyte/config + - name: db-pass + mountPath: /etc/db + - name: auth + mountPath: /etc/secrets/ + - name: redoc + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8087 + resources: + limits: + memory: "200Mi" + cpu: "0.1" + command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: "Flyte Admin OpenAPI" + - name: SPEC_URL + value: "/api/v1/openapi" + - name: PORT + value: "8087" diff --git a/kustomize/base/admindeployment/kustomization.yaml b/kustomize/base/admindeployment/kustomization.yaml index a944d005ca..c47adf129d 100644 --- a/kustomize/base/admindeployment/kustomization.yaml +++ b/kustomize/base/admindeployment/kustomization.yaml @@ -1,3 +1,4 @@ resources: -- deployment.yaml -- service.yaml + - auth_secret.yaml + - deployment.yaml + - service.yaml diff --git a/kustomize/base/propeller/auth_secret.yaml b/kustomize/base/propeller/auth_secret.yaml new file mode 100644 index 0000000000..b6bbbe4a8c --- /dev/null +++ b/kustomize/base/propeller/auth_secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +type: Opaque +stringData: + client_secret: foobar diff --git a/kustomize/base/propeller/deployment.yaml b/kustomize/base/propeller/deployment.yaml index e3e33240b8..c216295b60 100644 --- a/kustomize/base/propeller/deployment.yaml +++ b/kustomize/base/propeller/deployment.yaml @@ -23,25 +23,30 @@ spec: spec: serviceAccountName: flytepropeller volumes: - - name: config-volume - configMap: - name: flyte-propeller-config - containers: - - name: flytepropeller - image: flytepropeller:v0.7.1 - command: - - flytepropeller - args: - - --config - - /etc/flyte/config/*.yaml - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - imagePullPolicy: IfNotPresent - ports: - - containerPort: 10254 - volumeMounts: - name: config-volume - mountPath: /etc/flyte/config + configMap: + name: flyte-propeller-config + - name: auth + secret: + secretName: flyte-propeller-auth + containers: + - name: flytepropeller + image: flytepropeller:v0.7.1 + command: + - flytepropeller + args: + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + imagePullPolicy: IfNotPresent + ports: + - containerPort: 10254 + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: auth + mountPath: /etc/secrets/ diff --git a/kustomize/base/propeller/kustomization.yaml b/kustomize/base/propeller/kustomization.yaml index 408244dcaf..d21d0b929c 100644 --- a/kustomize/base/propeller/kustomization.yaml +++ b/kustomize/base/propeller/kustomization.yaml @@ -1,4 +1,5 @@ resources: + - auth_secret.yaml - deployment.yaml - rbac.yaml - webhook.yaml diff --git a/kustomize/base/single_cluster/headless/config/admin/cluster_resources.yaml b/kustomize/base/single_cluster/headless/config/admin/cluster_resources.yaml index c2ff64b3c7..1766e21228 100644 --- a/kustomize/base/single_cluster/headless/config/admin/cluster_resources.yaml +++ b/kustomize/base/single_cluster/headless/config/admin/cluster_resources.yaml @@ -1,19 +1,19 @@ cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" customData: - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: "4000Mi" + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: "3000Mi" + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: "3000Mi" refresh: 5m diff --git a/kustomize/base/single_cluster/headless/config/admin/server.yaml b/kustomize/base/single_cluster/headless/config/admin/server.yaml index 25a8ea4a56..e13f3ad156 100644 --- a/kustomize/base/single_cluster/headless/config/admin/server.yaml +++ b/kustomize/base/single_cluster/headless/config/admin/server.yaml @@ -2,7 +2,9 @@ server: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -10,5 +12,32 @@ server: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" +auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" diff --git a/kustomize/overlays/eks/kustomization.yaml b/kustomize/overlays/eks/kustomization.yaml index 4f64744c19..ed9f631986 100644 --- a/kustomize/overlays/eks/kustomization.yaml +++ b/kustomize/overlays/eks/kustomization.yaml @@ -21,7 +21,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.13 # FLYTEADMIN_TAG override the tag + newTag: v0.5.3 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -29,13 +29,13 @@ images: newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.3.1 # DATACATALOG_TAG override the tag + newTag: v0.3.2 # DATACATALOG_TAG override the tag newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.7 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.7 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name diff --git a/kustomize/overlays/gcp/kustomization.yaml b/kustomize/overlays/gcp/kustomization.yaml index eeb26bfd26..f1f7bccb5c 100644 --- a/kustomize/overlays/gcp/kustomization.yaml +++ b/kustomize/overlays/gcp/kustomization.yaml @@ -23,7 +23,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.13 # FLYTEADMIN_TAG override the tag + newTag: v0.5.3 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -31,13 +31,13 @@ images: newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.3.1 # DATACATALOG_TAG override the tag + newTag: v0.3.2 # DATACATALOG_TAG override the tag newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.7 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.7 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name diff --git a/kustomize/overlays/sandbox/kustomization.yaml b/kustomize/overlays/sandbox/kustomization.yaml index cfdb553c66..68eb101074 100644 --- a/kustomize/overlays/sandbox/kustomization.yaml +++ b/kustomize/overlays/sandbox/kustomization.yaml @@ -25,7 +25,7 @@ resources: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.13 # FLYTEADMIN_TAG override the tag + newTag: v0.5.3 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -33,15 +33,15 @@ images: newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.3.1 # DATACATALOG_TAG override the tag + newTag: v0.3.2 # DATACATALOG_TAG override the tag newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.7 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.7 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Override postgres image to use alpine based (rather smaller) docker image - name: postgres diff --git a/kustomize/overlays/test/kustomization.yaml b/kustomize/overlays/test/kustomization.yaml index 730d6af56c..b9aab66cea 100644 --- a/kustomize/overlays/test/kustomization.yaml +++ b/kustomize/overlays/test/kustomization.yaml @@ -19,7 +19,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.4.13 # FLYTEADMIN_TAG override the tag + newTag: v0.5.3 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name @@ -27,15 +27,15 @@ images: newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.3.1 # DATACATALOG_TAG override the tag + newTag: v0.3.2 # DATACATALOG_TAG override the tag newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.7 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Webhook - name: webhook # match images with this name - newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag + newTag: v0.10.7 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Override postgres image to use alpine based (rather smaller) docker image - name: postgres diff --git a/rsts/howto/authentication/flyte_cli_flow.png b/rsts/howto/authentication/flyte_cli_flow.png deleted file mode 100644 index 52d030076c42a2e8063cf4aaf38510bfc2e18987..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 41554 zcmd?S2Ut|wwl&DnW6l8sDke}t1SN=Bmq5UM8%RIl0<@JL~=F} z1Q8`lRuM@GN)!<2G0SuBegC`f@9wYreI4r_Kg$Zc_Fj9fImaAh%yq9T9XU8}Hpgr> zHnw?;LkCpY*rqLIW1FHeb2?t(c{k?|{Pl;e%E5hX32#<+vazjWV;tD4<{0?3$;t6> z`^3Z#7t@74*ABe4y`EX5Bz9}P`u>HgcR5B?_bV@D7pZ!7rv9OwlA2E2wa}XnPkIE` ztQdH^;>65tG7&q9#7(`fy_^{TSTbrN`fRp8S8uI-TUWD%V~y#z+)C5z{Ae z_SR4}JXjG3)HN`#p`BVRttxnm_w%8qT01N;%9sFfcGaRigO5h{e*i zYps*Np84yQ=g*H<*Ln_b$(~BH;av~(ar0(%uFLR1ooPrk#?rj*iSezf``V{ZKjwOc zPn4??OV=$M(yBLyoO`(2zUvTk@scH}=SFOD$A=p33mVHM%v{p?=~If^xHChg)OTB3 zM@Lj!sh>qlp6hO>zVKkzp*lV>E7_KfE-qty(Hlk2>0`p$2=O~%eJWnAN*bo4?d?fJg zgS{6O(t~_z67*sl(yis0=gyt0%CfgGY`VzrqMvsz^M|uWj2cgqrLL~7r{&I5?~n00 z*xKGLIH7#d|eRs|6z_LuG(0$DCd8>977Zn|pxv|VAIXRg_#5^27%wxDYs9>j6 zV-&w$R+L(_ij9qp3g%JOBgM7f8qx*I(l8+kGxzM;wPokd8k6yOo%GKiJ{&t`i>WDW zsg|HS655TC3I3CAgd^`jcdg=)BS)Cw5-vkqHf-2Xwo}xq@vT7quMG^FvuAxnLJY3l zw(Y9o5jpu*`o)!bsz;CBJ8|N~(p9TsF|kiJW}Go^%t*r8Fo}s5v;DFKW5-lWIjLM0 zv`6#o*-rOyUfqn-S1ihSdHnXs4^J&fvuudW=6@u%YuCw_*A|)ACIqFOs(ZLf)T(3X zTSI-WOZu;0zf#VAs}JN?o;7P$*SoN*=UiP=&8vC!vg{bwuV3HoG8C6L@@3Wdc&~fy z`y@l-RPCFtUA2h<4h{~CVFCXZ9O7^QhCMyA_DfB?j%AGjzPMR2*qh5|tL72)7`0b1 zl~dW+Mwb4*d`CyeD2v*J!hUXf-|)IGk3!r8Wmk=kjg{8bX^oE#R;HL$s->E%8)j?7 zYn#=lm=!9PL?pj^_pWbnu(~&YlAnPSOSv+CO^kAwY_xW&IkQKxN6@Hvp{mFu&dasQ zriz(aStj4(l3Q?VZpx>`s>g0pR8;iYrlW!T?8Eh~DO>Z=k)V<)!z2EJGL?~slV|C) zIJVAf9iKIG<{^iC5sR9R{&Fe5+}zw?_X($f53kKryMi2=Usdg1dvJ1MJoQwaV5`&J zO^2&uH3i1MS-T7Qy7s-=>^wO>e6tY@9l8R~_}-V}BoIN4fs zBIyuQaMd&zFs@V>D&i3pI!IG-XSjZrODB zJ|^Nm%wi8u&#FtamU!CdPmIQ5wtGjXn!LO+Z`HuJ^cIft#`}JpXU>G89Ei2hlgR?{zfEN;OJv%D=Vu)TT@fh zo2$iCD#qVgM;}}p**-eho#oIQ{3FM%vtnh;=11Xq;bQjJ-bbrGvP*x(z=+oRZqrfA zv@tfU`~LBTWtvmq*bI}Jc9FrC{F0KAUDdIEWuw=ZZ01Pa%+1{?H92xSiGT33`0?Y% z3r%@8ZIT@u?)BrEw|bXS#duwkYKUpqqMNMi*}9Of=aF33b@^oi`lrO3GtPwVc+%se zsek9<#r;?6B~E`hZ`GW`!{9Z@%ZbrUOhD`k2&wX`;*@Y62>lk3R}{n*@XjGq{oJ{n zGnuv@pI@$y(-Jb&!x~r5aXiBq?mh1G<+cCn=pCsi3Ou5b^mdjuHZ`?<{d!7#vis7e zx=#1Wv5+4>5d#k=BXKsHjNm#wi!fu=`~{7 zhn~hAQx5io0G2$FE6g*)G!(xwo1Hhxr0jlYg1rB^=W)%sE*1zZ9ye~NBBkJU?t=#p zj(>l#pl~OG)a%nXk%9_alTFLJ{7r+MI)<|&_g*&rkA_RBF~ke{DF%cjjEe9TYcHirmrgN5)$`+I@T-cefqKb#KxuiGS3;K5$m z#6RfyK3eJ`v+AMDoNanpY8B(_eq^=>3;p#1u|DQh-IEx#Xy%ci-4TeaG47KS_1O-I zI~FYrTY^-rv7WG#R)XrIBRl=vC9iLg@rc4I(9Sp=nqZKhP!=E>tCxMYuLID;G5FOrP$d z$SWk7=sqqk@n&W&L>7%W_}v3ctOwbogtX;a5dMYENeSiBH8$ zInwn?OCv!y`r?cQ92?nGHyOT)@-CH+7#lu@`{P?}?)srTP|TLIF7o#6+dqCbSm##< z?Q!6YP5ZiV?b^gti`x6mZo@rI6n}q+56&&i91%~xk)54gwxxDPnP-r*6F`7vO+@g$ zwx<`S${jkSXTthPh>+=Gr-j*^) zz%9(0q1U4uerQqa83hG}`-Ag1guQOuxWUy~;L5?mMIkE(T7){X=eViJySAc=HqE8%!qi zRL!2s?*2_DJQMweG)u6pw9=jM7XDF1z+hAn)fs*U^D@ot2Mn8awnh zS9`E;t$KWXf9lD1`-*O@5vo3GBJY}Mn|<~hXNF#vee^IxqO#Ax^tB{bIkM8-FCvq% zRlV=ROy0fO#yRA)M$qWm4@aaE?X*+=WupXfl7$)?8oFrt7O=>Om;KkO56dOIyOF4p%s5knfM3w zr9rCvi+sXgTwPFFS}Kn@rGazvTsP$|&nM#YraKjxyqbNYw!mLFbWuve#Vsz3IID}6=0In}B zEBh8|AMSShOIg5X0|Nuhs6*DdgBn-Ld9ua7_K#M`3q)fQ74{=A7B@8LB<2p?r&CM* zhG1N7_u_k1*fQ0gNB5kEzEx4qId$niD*(%D0YyEL_T$%!W8#1Nug(|AZ2=JS9Ou%K z5Etjn_=ePX^V`?2U*iz37BueU;dxab&R7zVrs@W)V|vQr)5}>FgoB&-+aB$`IK3v@ zq4&PzqmGV~7U@<^ob{0fCz9r`-u2eG{mq&HS2|>tL5U)w8E1yU>C@l$cO%XDBv~Td z#lL#>YC+==;${s%I!6N*a^=zCGSL?C?~Tnay%OCSXIedtyK55_D~IZmO)Di@5SLbn zAi`e{|Q@ol9fvZ=qR*O;!ouPjs(0OM&7TJ!n@MY0; z4Gk7p;2dH$;k+7gTM*Kn933&>4PR6mO+#EY#)iIC{u=40dBVO`L*dq~TfACN!grtj z8U^5H-rJmObNcjl*CUtbEU$5BaeajQK5wPq0YF1$Y;5f8WqeVH5Z!~d2DNX3_J~+F zi?eT0dTN*L&pjAXYFVF> zZuY+Y`v=?n3?#pc2nq5`j=py2+n=@@!8an3w@itL2| z)hVYx?$gUSy;mpglyXUn3;?gfJ=_;Iqa}Gxtk2Y^TBJJuI1ylMoi`HcJlqqspPh~| zvVety&2{t6X`u{wlAf-KDYN~ z5w6Ca5(hA)xP*M@jfcM+E=~g!8LqC+B~jH_*$b66QXm@5Olu(utv8)2Y(_$(4(oa!;wu95q@-j* zqH=1%=g&r#%{f{CIfOgJB_!HA0vUyFlYRm{NE+Pv0H{|H`3R(I^j?0 zv-H`W_^dlUNsIg1A|#=tklDKr=u#-#j4e~TBzJj6(QL2$w>Y&`I7cZuM;(o!UVqLl zn>KGYJ$skUX3>nRV|QN&^{@J`Klb;OE6%1Y%zFG^$Vf?Yg9hLQX!+KhyXTQSPC)Cc zS=aE@xhpr|MbxOBNe#Q)CvuRJ!_J>SuMjtv?ffr`{x4-j3MnZm9r621ykz~OIZs^; z3q9WT*~Z7GaAUbZ4NA-eOaQ8@X3w0dglNab z0D*y;|4Euv)BT=_1v>5>Z`T8L2$VTB=QydNr05$OszE>*B?W6ttoZ^4-kA zW?ZR%71ukzIArEh+m%S5nVFd@%Pp#72H#s!QI5j-+(Tzy0fPj9m(pj?_Tpsxz*Zo~ zJp%BsKt_L(?6P}CChl)(bF<{5QZ}~FwwduQe&JqT3w^dI?-R9dRy%a4c;i%=8wGtY zi{dcN-6lp%5l|)1=-|V~LME@@J~1x+{OQv}q!}JbNqu1IYC^tFeMTdrqxY@0Cg<&0 z=e|MdwJ}0lJc=qQyMQ^%`8y)~^i?iSpC|0lqiZ;FZPCUQi(3A&UjNW>QmUg$%a8zn6*1|Y_K@7}#@)sh#lm26^$dJ2h434aZObBaCr?jbWA zS%pdsRUAgd_KN~HH@6uwW1Y4X8=JdNz^E%D%eMU%F(vrZg`8NQ!@rXw?G72>1?Lp_MY~O|VocsA0 zMO`F7g*lcmuX>C(7mv8O?*1!t-vOD$p{(Eg=FJ2-KK|19vHJl=bxT z9?E`!8aUexBarg|H>c#ynDW%kGQN=ls&s(`o5A^?#A5>7L2)jwtn54Psz{ zH7maQ;YWV77IkCF-`{C@KiH)wbB3D(ZsFy#z0L8!UdnlSdAn^sUz->ycjx8d;aT-& zq2dzREnfsN(88Pzb#-Rt!4jQTwVAf2-`*$f2NS23sGpl+STvt|(jmo5#Tzh?JIP}0nB&@~-IgkrAZqXKoSTYvs&Wrl;nH7bZUDMmE# z@$!0rJ4u(l({HPF-4hvbJI=ki!=ARM?d_{+&h|8lQ3hfzU9%>hIBg8en>)5_Y#$PB zR5A28EAOzdu!$eH-8<+?aX$O@?Zb&mFe*^AaxqX~3g@kz3z}rc{MCmlYxeZ!Y(%JP z%63o;*lAgT>TT)LrCg$-q98d`0oAKR_h0?o-ah=ccFUG6hmiC%b#()GS~thFxKEBN z3v6X$b5c=L^X6LFzcO4S*V!Hjl41HLkdbUP^tG-&+4S}V_{Uhc@lmFCkHZWGV)I+f zTF(ZrzZs<6ZohtROpKA9*!SRGX?UJr#fzRF?cct|FXomziKR}g53O&2&lue31&!_C z+T`TqD!?#Q==QiEp9(kV#A`50D&zxH`9tNzG&~UU>f#cWLjS5yvy8pDO0=ZBTyf2A zySK54dAztVJUzq|1_RbAJH+!1%c)@IQtoIl$69Ck0Rz>nx`vl5UYz1Oc9wrTafYi?2yTT_?m^A5CjQ1`AxFA0NOajm3&Fj~lj*Yup z7J(iGLXRB!gKeFF{K&A4bLoV$bt_Ob5q`WX;n#pShl|tZu=9rFMKkdAGzkICH+N2~ z<(;+k4#$NP{DxojJOA1%V00WLzYIubkcY?azqop6<;rN}lNj8IQTNH-$!OGY zvuDq)fHan{)3Uw;5qK2UUn+=7VLv|gM18%f)91d$0wZR;I?=!Y({(r45)d-PHF5Z@ zNAf&5I?%au%;8(TNSVRG3Cavq{pbDalOOD9(!n_iTd^6SwoA2c(XW_n`ZEt|7p1Yu ziD4-%zGWoupWWPqz(*~@v+KWR-t%~j*kpOAr}Ni6nFxU z+qY}+0pp~jpf!IAYYO}v9WB9n3_EyR<2I+nz_S@^I}jA)hO_sBI%@my;Q<)IPB8Ks zh-hE{{r8*RXuP$^L-9IdcofK8l3hr_N!DeefbrS1@El~(&z+soRFtB^_2tSP`4Ye( zWa9bh(v>I{rd}v*@adCJY6xg7Rw&mhkMkbA_KD=$w$+1zc#=f7kit z#}^3m2a$BufUnp&IAZ$S%VQDx?u%LnLkHtxKn_s{3Q4hS*y%dj`6$bEENzowa2$x8 zV%(NwlQI=RE&J~u_L8!M@>UIrzbyYL;Ngb4re5oYw8Kz`u!{W4Y-Y@u;pF1djcT1D zH9~46?n)IQr*WeZQQ?};sI0)v7?jz9lXD_(;Z%+G@#DxugSE1E1R6qdQWGp6yuA}7-wqlo#mOEO4t}4rLOX=aXoLC#y zN|Tkw^kM0|ePF&b)h4FK@G@Cqu`6RQ0XQ`%ZvH%duKZG$ce z6;!we=%Je>wx+kZ>FnLRcV|&|V`eg77**k0*RS8pw1bR4_^m;csu5uPrOTE@LPVpQ zl?kvzay38akUgR<{; zLD5CHw?IA^g+{0j(BdOtpg(6BpBLBVOP5yMW(7X28+4A&+wj|N}PeS{S!9nKRW<$QR}Ufk+7DuGonQC+B-jQxxKWybzkg$*2c7xKg0%@SiA3q=Iy1e$ zRJ36BnS&l80wr%^)M6f|_)*+zOrTU@6kKzwKXrBSE?>SJ8R1B9aIjg;9x*XsPkCO| zNQJDi<>2>CnEq8IAm=fqzvq_zf7U;@-bM zx<z>5h=|8TV~1DxQm_)t@)hdQQ$cY z^#F2Ei-rRV9b)q9=R`$CZDCYs8n{M4gn9+~4Derc-zy@MP9)KaBQ7X()uTPmDa0^W`&3K3o#> zXNo(gc!??m?N$e*F6$IzV|!}&_$VKywq2)=kaH$>C~R>%V2ZgUhJp|&7TBHSLql@a*h2OgIF(@Sv&FWEeA zUvyy|SM}0${L67Q>jLqN?f-Vir*vfh@UZ5|lP4onNdFoc8>4XUUlGi8`O1|T6kYx` zoOGg}H&b4^`TWHTg6W709HmD>B;)g3$6#JG;n;rcJXX}nxIJgT-UFq+!P>fa#&kQ# z8h8Bs{9qZdPCkd_f-1ndCh!n13jSI|!#l>`hz0xb^g;~yZr?SmNO6o40L$nu@_HAF zMGZt9OW)fFLm*3PsrYkpa_VeyXi3#F2$P+MP`zc3e3@5AM~9uC4xjI4s|NGb67$p; zqzyCu^v!oxrqc42c9X&6tos~bou}->s}3ign4uO}L4Mtg2gqcU1;9U4q6V@5_}Pm{nvkd2X{oosRw>GuJ6?--S%IYPs!xs8PB|HR zXXC+o&yqlKrzaE!Nma*;N9CEGdCB?trCH=FL+O@)s=E{hMJBYcaNL1t9EzM{DE~co zLb7F%=6D}=LeeWn4mUk0v&KEOW@K|yX66odt9djixFZCEPZe*HKYwCfB(6(^$rEO(eEUZ_1f zvIYpCBR;)Zu*MaZIF3DM)_10xnjpY_-hOEfFIs| zcy=)rWR!52(B{MU!x5Uy5qil_gFCm~1r36XlBw?F8SerE(CE%*fPqGgS$|FQ(j1O1A=#710i{PJ*%4d= zaG#LR#8bwJdp>Y|G47C1{Tk@-D7-%-#e$WhYLl4)o8`1W|BTYjv;llk2WJ=3(MT7t zPRbU6k8NFI1B3{Y9>8-8cY}@#CsUd6jvYIoVD4SePP9llb&1bfRi#;k0h#05l<%6fy~|N1SO6ZFlPMydMTI%frTru!p#(_{u6o` zI%8u~fsh4O&3|t7Y&zvbH{^r=Sa*X2snF2S`YiicERq-qDHL1Ok#J#Z)*{=gf%|wW za9k`L7~06zqH7W&PPXLbet8oV3miePmFx_MuZ?VhDxrb0nOL8D_f~*Te0o$99?msV zZn_w_ESKSgGNa8*(yKw4Z{g>EbFaAVIAqI*fUcMl2SB3Shc}1Ve&|c8sr=e0he+a) zJqPki3)8<6=VO$rZU6K8aqwO8OrkrDOMN>L!BAw^fYnx5!-A|{%aEnkU3DFJchGsb z$$_Xrh-v$Q-FDP~@7JO;-M8;AafRkT-IhP5PCtpcTbOtFzOXV@C4`ba zsa8$ey?J9j`)?DyNQ)f*O2G0U*{Ua$8QpU%c@|(q|*a z@7Ifs{`7D>L2@57HOos^HDW~$dv&z6`7DX}gYEq0?PtHf59Fs~XD+<^9)V>UgAy9Q0cR65a=_{Oz^z>;nX0)>|pg3Yh=rsi)45Jc3GKi8QJiL6*zja-7|WvqD-0(tGZWlr z+kYG*))D`ofL^kQaDbze?@{S?ZdCoqKd%WKB61q6Q~L9~!cunj2>0=B19VD6kl+c{ zv>Phf?Z$GT1WerpOJNK7~veZTEX5HSL{#*P8d4VR`pn7AGd!J~>M|SVD{dLvs z{o_!{@qtQVOWZ4E*a%+rwVI+mXlxMHNTbpu*3SxTpr5q?j-Y}cR603OiCcgI(~Y!t zyHOOB78q5R52K)}C|}rjRlOr#5w!^^sl6=*u$&a0CASN5uUeL!8Tv4+BQ4ykeyoj< zzA$C>ym{&fr%8Megb_9BY|g=c>)cS4$Z9@X*Gg63-9M*KkAYQov^IYtf=phtPM{&^ zyNwtd1|W1rS-0fT3s>f@d`M2tuXV|N!^3`Dsb-b3;O`Gr4&^xYSD{R&+7PxBr3g76 z61m_F=mOawls0G9EM;&d{<$AAH`hw(19vip#aZi=;Eow?S11^F@zdi7 z;e$0g)@D!7|1lfq4`xvqr1C#l3p=TW3QmFAV4cu7a>^IJxmW~Pq@|FgQ_!dp_I&UL zNe5fS#76GeOMU$bvQiV(2S#lZDIeh071QUz&p@g_=msX`X;iIh2w#q0`%$1|Q7H~U zLy(uy&zY4D;9;IWe@>+-M8gQN@)_S6K$1ZNQNYYlK6L0Bu{p3RME{SF>-nw)G~=YT z=^7biNrHD?o{98KB&mp1<56H{KRc!i@ecV3|a=`P+NU$QLX6%VPs;G(FF z#jGYs(slHX<<(1<-g{EpQw2$MFy5kvX1P9M1!fsHg9KT9mv168o2SiQssyYPr(uz` zo0VX~1l)D`B3Hl86l^jOJ4b7(tQwiFU`6YXBGU`|$0PPPDtO^PXyTFt% z!~eyabRL;02wj&f4RGXctK-&@>j9OS8~^|T zN3KqUnB>f=pUOp|vD9eg;NX{-Xno%vk>YRK2x6c&7fz-uG9Cj+jc~+%Fz?>AtdTM| z0F)0xvlzTDRF)x}$HAjY7AYWRXpv71L*V#9RXc;Vc085~BoLAUrhzdv1PeY9L2Bj3 zKA@jS*mf;2v7nwRqfYSr9&`HRbBjgfZw;3mqnDkQ)D(6rWYIqfrt;zuKRzuL zIMbDpE7;)*m74se3M3$0Pm`4(DWM-YtW@A5_q0cJcJBC(xQi6uuBPmCGNWL^o$UMk z3T#rnVaT4}7rw0f?yuNIUQ1W6=3~HgO?nU;o8X*ryzR7Le2?H97SlnWg~{(4-}1kS zp7VBZ4`P~pII|GYX*HUW(9`l3T#s!QT5`Q^jr6z29=y#7@9@zhemLOyM>ByY4j{rg z4}FckcI_GqP>CNv^+qosS+qgP0VN`b=a$@t`?z{0Cb*$c$B_wO>f z$n7uoS!3bpOG|HUy5~H1p-6$`k&&d70Y~1V4f2=dR2NTG3|2Ik3woZiWb>Jmf{V7! zP<#2ufu-Ktx`(ey4QU=Y7W-k@Imv|BZjaodtlnYQ#I_&byYg!H6;K(hL8jlQyz|re z*?bP-%(xHnIXYzo1qE%B#~T`3e!65IOFe;l1kuX0H3p}9I(>uEOUpfIex#l(dB>A0 zWbdqdS5{WG3|NkBtL~)3PfQ)76<2VREn7?I8=@ylh442$Vw3hQN{@&8lo$juxIs39 zIf1>dFVNfDs1%^?>Aa<1*JVufetYm>9TfBy6MG!?Lr>vs8E=1B_hb$pjKu_flAPQ+ z0(hskaGJX{bSrJZ24uA)dqSKUHx_MqsE}&epn-Z@T}jC+o;;JH>p{!X3-Cx->QO1w zWAmrSmgThx#Jo8OXmI0-KbZWOE8`vTSGC8YMT^piqid8_Qd+oo13qrWv303(&rg|Y zZ0EC=xU;5$#y`!-kN?%R9Q)5M-v6^N-oF@6Rn60;qL!AH8g4xOz+W(127l8-<6kY) zA}kv5UoS>UgRO!CO6kkDQSdha%*GWOZf<$h)`*EGXh!MNVL6Mzo6UvxNhFrc_?@(1 zK79Bf7!$Px!VVcXLF5XglLUNhPfiPKAQDcvmmX_$R#y4+z(&~K@ z5{{&xtVM&Mb@n!Vr7`Fzx%8Fxa&d7L8woO)f21$*JK~{6nM90Jg zpoA()ji+##0dd?1cT8S2Cue8sJxn^+3kijQ3+^^`RvDX`>fEw~tqXiHrBXt}DJjCG zJK(^g01PPy%2@I_jmMX<-0B4V74V-kME*APY%v4_%1B4YxkAwjLpb&>O2Hg?;EDN3 zSBWOit1Ppk`HET~3RHz;2tWj@N5>n&pUR0Ojr#lR#a`v_s^U%$?QR*FK{cSd-#c1gepE4lq1;XeW;UA5s2qMXMSjf#a>B4k}} zU*E{Dt79fm60r6Kn8E2N4FWPhUXrYX$Q|V1AV~oZBW?x=4**bA>MVfyj5sXno&u4+ zt$qGl$zNBFe@X>;MwS{t)1&o4py?3m)xmd_${nQp-Sqd_oE!U=?)=5yluyb427kKx z1ya8r08Ei-C=xD3zjTcm+h({j$zB=95M!!7Qb zM~>Wr58ET>>q`vw!CR}}!6wEhV8inYLI`m}u46-;(Bp_uMC&}+9b>x;Al?xel9I~b z7qS3V2oZyTuL!Fo)xk-1w0aXx=G0Yo5B%kjIr*zyO6GYsDASYZpQLsdFtBlCQ9^OU z497Z)LZw5u7j?_P^Emf$R{8m7Dk*-DqFDQfV&ViJ)ht6`px#a zFKT2SoVaoGrVkf*_6Bkp@IJ!r=7E9)WjNg)nef4s)?o*4_B1pOvNaaGHc5thb%^>& z&Sy`*fB^hd@AYpKlg5uvs4Zkg=s$+p=ESpY#zAjWw4z1*xVBhGI?QLwFC z3$dNO6xszfnSkJYpS4s?1ptWk@Og`jbu|8(%|G_7P80bQE3O>x-k_93 za1YH*DfTRRpY6l$E5{KM-t_enBZGe7Q@ZTDyEXpNoAQ`Kw{+Wf*HJsV-mJM;tg^H9 zi=VeZR8r$ZMaoP_ko7jw3R;2M2jHtAj$e&G?r-HL$_Ncypk*In;ggXVaVL#`me0(s ze>WfSh9Wusb-;VhKQBET0faa!*fcG=Rg@V2NI`0g`&^YTK*Uc99 zuU$3F|Jz>ta;zJJ&31bD$MA5v{VY7qkY`0z8te1b|7@T*`mb$3e!n>3j+i|P%Op2r z)y`97JGi;^Ni&ua(B#nsR~XJHkvYWWf5OmMd@jC|er_VQbr+Bb6+pn+j0y;_spyCn zj81-vr*JFjM_S@!-Y*s~#+qf$CYA(Ft%%*=ON9OBuby@_>k+K9;YPEby<4 zD#+Um<)$>rB7GoO+(`v~kXmTHh$d24yjpoes8NWaK}1kT25b983@>DuyZA0tgVTPL!(T_ti35pGPy1eY`r-Q zCS)?iu&STTOVC6F27D`h@nS!9og)446=9X*Z?ei!p^avOl-G(lB{GCUUo*!R0Uoz* z`En5<02?_)cRHOrca*~w@f%bZ;3oQTtr2%rLKvb9-;>!6U5JZOPX>9SVYbT z_5Mu^e}8|1o+y|S(35gRYJA{sL@)EpW+5R}m^_q2Bwd9;A|l@}H5)^^p&b{n2;&oG z0DD>O2iMQWd-L^cg?Qae6(m>{@ErC-U$vNI|lV=-}moI&dyobtbmU} z3lfonML1MoyA)7$>#2JY5(Vngci=+RflAP5M9IHkNcnr?{ux(eZ$G$o<3=TViQBrm zVj%o%LB@jA>%~P=U}9pzxhwh+wmNwLpKN1A|B3~GEm&_BV8+R{G|}gmPkWBgAX<0l zeCk3Npe-)i;Dtf2OmTI!IslnKS_o!{1)x6P@qJ8uK_m++OHU{F31jqF}N6z5-QPa7Wd)9u~LUMYu5Pq_|P6m>{@#@?gImA zO-P7{s7FUfkAj5a)|pGK$*mXqQ3n65H`b!h=^#aapJB9^50EH zH;|{QSa?_uhU)P^)pxn{fHU!ck;oT)xosJUv%oq&!NCcTZ?*~u(8PQTj|;VKNr1+V z7&;Cl1;&jyo{H-Dj%j~<8YH7MSOzjPIe$$m;(FDUq#FyD5m6M}TeexbVY?1W5$st9 z6B662qE%z`bF%>LFnWUl3DYP*umA)vmh}u;`PMlxG#U+?%1AS5vqM2(XwnLseB3?N@gf zTWtM?iPVs3`-rSD@Jo@=91B%+@awZFpiN2WRKaIax5!Tp4Go!tyXSg^h9LOJC@ShZ zcXGRqwE3Yei|-6;>4OU_T-=zIk+jEpT35_*JzaVKu1*ba0cMWxE2$7Zx6?6xpyXPp>U%zWx=iuGug0qYak%09mR z)W=w#Di25AJCLH@K|%2?Zexd;MuPstaz=!QAIi!){1ZZj#;^$Xw4r5&e0P@%vw#OP zhB@#xxtw?BPHO#U0Q8ziazkiSJLRGACLBlEI#&f?|DEhlYuW~-g3c*sKa+NY82td+6rZGc>HsImz?Yjn_!tOE8X#WKlMeoF9Z9W_I9IM*NgN~1Kk6-o zr;%T!{KW#PI8V>HkOhulu5t%!>bmyy^!(lhDg@d9pTI7jxA;jiA83>*Vz;Vb$4|2; zS|KG?PhBAXz)#_~cB(*Y9&FSH}- z?ZV09iZ?Q-;RB71q~@BzWg^F5B*l5>xUGg@QcGtCEVTT7X{K#EZKqLET>KXPo%Iav zH@Cs*f+L6mYC(A13vSXbG+k|ZT+S&dI|#s5W=q)E9&C6b6zAHsM#&Q%J_Zx+%$r+r zcYLaWrl>W(G~gjPuQFEU!>S%WB>16o4LQD^iUzi8(u*9VLCnR=;~(se=qCN#-EGlL zIv-S6?=Z~yd21zg;a6v|Rd|`r_~o;jX%B25%&t51CskD|>APFzU)WHhhSG?*b&}|7 zBIv_7!_^D7JbbO2&71V0?IRf3G(N)Yco=(1FG~Lg)`eCy-^bA7D`fsU0B5C+H-M>C zCHW1yQO8NIqz;F8d1>ya@>;*Y$#(Q6yGMp4RIwWD(L_K9J0u}EClvd(!&t223m9d^ z!sHM7cz%wp$AZN-ezW;P2Iry92+la z0Pj^SLK82jNrKyRU=>anZh9EAOnCI{f2|0i_y4Uj`QP>8>QnH_f*6nlC`e1vZu{~j5*^yUjG)>y zC&aaYm{@T_b6B8v)Fci0=hW)^xs;{WwwvUjQv{0lG7?p)e&n{iL zVvaPO#vk_jYjds+Ts#qO0)LZ$Q`!Gu-)~|PDf#om)OhSQfg(G-sTp*bkGHqlbRWp| zCAZRPYoCA? zR?s<|i)?TQ1s}EZl5u?7wlW*Oe}m+QnB-@rZd@b5 zSEwS^x(+B`cZFR{1@{?5en8>vwTaj&sx*l*3PkEVC}^nv+GW_PGzo=*8NxV4Alg5% zs!EOUHq@UQ&`ex#VdeI}*(3T@h*f` z*=~XImJ5Xy)S^aNIgAeR2q>?N1SzIm{*oiDpx_Bk52;)SzI)eNFR2^XKV zdTVqqgeGchBI61PC@6~9cMqeO8x5HpSzCJcJ-7|OT@&;vx(-k_u;N%m13zR^?>H)1 zYV`$&w6C26Am}M~;XtHq;$R-kXNHS5JMKpX;%z6AL#+oujN;=ySRX&(&|MYGqG zmy}f8+hOP(5+ZQ-RXC7_%GdzPuv=^PP&4-9$B*}Y`}ca#+@&^af>XfihriS%wIT~<@dqd=5L7FG%mg(O z^(~Ezj8>q;Eu>w5FmkJIwW2*yl-f7&sql!1m1Lo&e;S#Xa6qjRfvE3^WbM0K3Z3S{ zV9L#)?{c8_Dg;!r_za@4{&iWI$WTv{g`d;;?hzf+WW9H*9qYAg^}P&T>16{-OLjTw0}|d+4wWC2`ny>1SHxhC|H48vkUtc zoI!uW23w`Z@fDn$R-POpCA-kBf!uuDbCu|uds6Os>b32dSy!-ySzG-?x51WoK3ffR^St==YrYEK<11uAN$}M^Oc|2PDw4ln$?VZ#lOFjeTu@@X zEG+!HDO(8+C-%jQA7YJnU^9VQ4ERIrWTlLMtC1rO=nv4B0S>qEb5BnKhOqEaHc1RtsbU)dA1<^XE8+I40xwY0 z{oqg=buFM;0{cL)9PJg<>Vwo4i)i5Ma|1=J60}ehr4f*62rP5oYIR2LpqR za)odZ@unPlHL)d+f}Q2&&z}$G;9%$(Rz!QGSWBS_MbN6D25fFhkpsap1g3Uxe}A(& zXYKk_Cv2)pn|U!*P=Ic0NB6N3ZR7~{U_Ew!BABcw{^LNB;qCvbs(-v%)L#L1R=qp@ z0O@w0ix;7mBabv{n+g=!F$DFY2*5O4gR!Fo0Z`6Rxjy`mwOu0I6}7IzO^?8icY-qT zHAI!*3=YP}-=8W+Y6m6M-u4sR|L2a5Ef`edPkdeav40)Q-Yjgk0n!`Qgb2c_$P@Cb zUC`2!;?S!Po=*Y6pFB5nZsv!N?SPpIb+RUeaIDjC9Bp4x?c)^5IdcIQNp4CC3NY)H9dLhBH;}t+DpO*-fm>*Unbzs|3 z+^K47--mv{GCYEFk$EF8FJFnD70eVIVK_hRJr0;ID{6sw+hl!AGZx-h9X_6MvrTtxlz!!Vl*!po5-{^f@L5uAEWx{ zmU`>q!-v()&13S=Ud_-<-?BVqGXqW8qT;>4G$mza)%a^$2!h!gl~$Zl7}QbnlVh6t zgvaW9xskm*k+dolh+1yVD7ncA4r@$TQW59f@C`bT=j=iUNlzK7(5)y=QL8b*JuF_l zxC+2RrAQJa7dB*(MMfbS2CSUM7B#*ex7Q^lD!5OfQVY^gg5-=J8fbK?dJ>Z;jd8*L9lV`gvkQ<5 zwU%sBc#9c2NWI(z%MJMYPF(8-YC4R2;X~f|S2!sU6#P#<_2L#)VgFF*S6eZMWeyxT zsu9&h`&j5f_Q?-twigu?M7%Uzf>tzr#2@l-qQJ*AwNtfk>`FCz```k&Z)#(Lu_&Ud zR}>dPLL?0IRxx=PCz1&viH*!lG=x2xAOiyVx4Ptmu%c%mTzKRzmxtpe45+g+ceq&< zi=CT+J|K;tZcu9nIh_s=`vS;@Xh)%i!F@Xyha7EanTSLupMI&IKrQ;!V?aS@-&*b< zRTHgx5c5I~vjuUGTW>u181`g9wb+H^!L1*HQ|dxZOdAu%1&36OrWB^uzP-QWkZ4|! z^`INLF!mXw3r2B~HwL91L@h;{&mbe9)`gSKhO7zLol1^2h}YdDuCl}q{tEnSVeGN_G&C4*bL=dD%;K4|Gm!rXE}V8)!WnNB71hMC(NULL zHHKAS0o+WbJh_Vy;1~B6Fayz!0chgua|@{_*_=N-RzOn>X}8&RSP#kuwWa%hj|)$7 z$A04=uI@qnMXP~%rfH&jmyDB-r)Ya#^=VKmG?pJpa&HCr}GlqQ{m$$SXm zv;=V!)CQSh$+VAvYzERBM1mI==?2&q(wBO)6eUN(-zN8Qy8S$EY-Ds21zJSSe`IWc z8K@|uepLhw+A~i>!?!Iy5tfAX zhjW{c<^U=IqSqtdD3V(oQI6c%*a3+)Z3pefS2Slr0>25Oe9KY}t+4Ps`A?W@v_T3< z3_yT``8cq1M<^x{=%_>x`bUXS;wsHWxL6G&BM;Ah5-%IBvlcl4$tIL6_V|6W^>Rb} z5p6lAO1n>Bpg@31k^BIWp7#fsezZdf+k8GmRRUs~lAN4}ahtB$?4D&Iq{?Z6cPEL@WHToH=t-gRLH=UaJ!T%+p2 z%%O7TZw777w9f@R(TWzRsd=H)V$JTxgjE05)WMlwJq9L4ZS`5;^ECb`>^je03}KH; zOWj#)51#m={S@V36*i{U@%wA`SA)q>KiYqc@*@*PVl3@Gg)~hY6*n9`nL0tx~+Oo5<&(=(De2AJ(6a2s(ZgJtc7uR9XK>K7rP0bHm zbtM#ll7h?5gxTzCSY{4-_vp=S<^NlhmCCa@S{OFEgKQZqNyPV%^x%W73{+vdmR^+Z z(a>vx*zkzjoT>Q(DgKlTSyb4b%Hm9|5&hA~1dM*7o2hgiH|XC?Y8c|+WpD&wKE@#R zg46Sv3*N3EHTHK?!qcD^F$cUwgF~;`s6N)-v0=mWziGVFiZf@=-f-wZrs3rcC2J=4 z|NhmWXT2XXmi*5p_um&z&8hVGKgNrd5!?6)t(3pTejF!bKT8RxLq7eleFZybEmaH_ z=K*|#3FzIjCe7W``oU*$>-gDdx{MFjQdhzI@ZLR6H>ukQF|>KW2_+h-u}Fudp}cf# zHYF`@iw7$}NFem806nDr0)x}rsJ03f7ue5?h=~Pjc9Ri53U)IxBncz5Aoa>K;bWp^ z?-Skn(3G|fI^_bmZ{4-)XJq**bn9UQk&^J~y7rLwJJG(A9c@ehcL6|@EQc!?49Yi< zoa$`EAuWMDCyP1l_yXib+d^;YuIQclF9RLR+=A2nz^~C`?G!9%>;X$neq^Osf#CjA@B+0=-t&g2X)PQZQvvPGHpxGZR->-b-_#q}~6tD!umh0rZ z<-+SHVpupAn!gPBi%<^_Lc60q1)1RxpUmLuhCzq+GQ-vEaYe%id4@ARBXVFV`0XpC z)e%@066y#Y$NpQrdV6_D`nhGxm(z~76xtD^YoY44!EpdrKcfP7LF9jM5m?XPx7p5IbpCLo5bj$Ot61>W9ENu&coL>Uv6B{npuLcFKl7hs$aU7gMawH~VMQ=6RsrM&Zw zs`A{r{Z7=F?w7>GB(_AiU1CK=MT{-kCN>aIP>cmd!Ah{8fSOohVnYF2KqHC>YOtUv ziirgUjS3242}VVlf>IPg;Qr=DPtLvHIe*+SzI*TZ#$}9*R?O3#pf&)Gl;qD} zc0!VlNLY(XJz&EIryZ3lx9OcZ0@IPrDvW;7&}XRZ6J;z+UD@B?x3YW|{W}vn0f`MI z+r)1C#>2z4XDwe=H=bmaQ|(Nz%=7Eo^)@>s{m-SY&)6yZ5QZ%OHLiWSL>uHs4a07ao&F5Gcqhy!CD}G74u+Lo=ZwyK1nKR zNmQ)qCCEyO+ag;fJvqh=;2XLgQK=fKy~>s&P&Ltn)_{J&$6`2BP7`SxUmie>=2fY6F z+YZ=dY5$%%bEf^qCrFdzh(-mS%#xQ*0x!wr87>%EDNo~j@8s#Ly7tKQQLs|)jH2Ak zy6fmRc*g~sijv--a8UQ#6va>vO5OQdzLpJhWH^(vIMf}MXO_ATz0rj{QM)xuV+(dq z*)tG}R1hEFjBrx+oj`QDT|5;hw>Ra%`E}DZDrF%zr?k(fQWyeT2EVzT7%caz)uUuH zY#IlTU8ip1|D=%fr+wan+f*>c)PK3zVIiM&x>fk-(IqSzKUM85=sT25|Iddqt=7Lq_(iKEz~&$dr8lO(f1Y)bQv~tmmnhsqC>c42?+^%cm5%b5YJV+mlU4; zT6EcNPzwgj6x4^Kv_6b=?DTuo#vwOi<4>QS1ihbD_E3;bT4T;`9!=Mns8f!N-bns; z8EL3m%)})DFpE}9aCMc2lh*Cyw=P)E%G(qgQCNTniKHCfc-P4q6*gy`Zh>`2ZoPdP znb!ai?)ZWUcSun$V)nNK>q)nN&9c}QqHY1VNGuLHz$L~?#I*I^fr^U|HzOwroanrMji98|2ckT!!|?LeCu^88_Ulh zC)?nIcrxnGGt2MYJbTu}hiw4Xy#HX&z1tuJi=y`j72Q9N<%n3K1RCO+0}&64w24+O zY-MniY#Lf%ep9x1k^+PSoH+YZHXR^fgwY44yt%r!QDMKrdF7TZ4@I9^egftSSc(H$ zFqu~pmWE9_)Tw7{6E(sc=gZi(Ozv?aUm8fPQ5d#=Fk-Hs`R2}_>ljmWSqegXPFk zaYluBcuD-kT3gxYhr}LiYC1x|V%AYCw4j zmfrfVmI^Yb6$kN65@H7hi!{|Dq#@qhm>dLt)0RAi4HfXM<@cM&KOjs61O_(6QL0h- zpIILVbX*bNVD8sot(uh~J(4Jn{<~jEKmuMoVC7s&Op(&aT}6yQj+YI-9G5-Qe3y?O zO>BB$a3th~(BE7AtRUx__yLEjSvHrU$u*?$s!s%mfP3zrk;aH@%=09}IfGYmXQdQHeZnG^BY_tFz zED2TMvzLE%rB=1y4)d5Kf={P@8PsTm4|^ ze#!~F*jEG*F_glXDD#_Z{ZJp?O2ZUDvcAN`@+ zAa^-U4ub0h?n$nh$dd!6ZrQ#Q38Bh;F=TjHYKm)*Gh=KS`GDl&?2WS!s2Vb>0_+X| zXA3oL1n1hR#XCD6?zhdn6@Q@xcF=qS_%m zV5r=uF*d9z`l)~qy0#G{l!!dcWKnv!-ejPmanM>t6@|0};OOwLNgv@Ct!hM+K`}YN zid8n3^Yn1ArB^2=yKtA1XPNVbOoab0g%fgBWFCq%XtNGTVlbzT)46dC)XqVJ0f5v9$AE_hr)7@q=faS;w0kUHSBMB zd}8X8p{x$NWZq+jwWq5%^tC^vFDu;O$gHW(K`-C11gf(9=D0zn0&k1{tfpbw%I34r zUOS)Hop;V^Q{nr?@R(M?TiiZeEcalcj*FtZReN%n&`%~e7mk40n1@_p_cX`q%cN@I z^psNg{jyia+;S;YDFG6dYj6-xTwij+FGLo-l?*och>1*qgDUtD@` zUivz05NF6TP@-(B>e|iq z&~y!*P1F@l!ieAe@29VrFVlQYJ8IMQY+;hl#!SuOUv}Jhwf2w}mv@hJPP#IoC1?si zo1guk)q4GVf{6LoBP#m`oMZlrweJ9)Z2V8)$v@M74sHE>4E;GHxc#FQ#b6fvOhcsK zXT5M`DP5>sZd~#Ou`rq_hhw&uSAEwT?tK68yGI7FYK&*$?c6BCwtlMwQ6|#m&z_(t z(oddib!a})$dUlQ;WN``Ro18+V}6*?6^pYdj+>Yx@qq$8{Y;zi--H(Q@4%O# zU4){~f2mqR=q@_px%qyTbHmfSF;kA%;Khc!o|jxzsl`2*nP(5v5>#*_hof(DW?+U_ z?3KrfxuulJ!>O}d7S7aNx#i6cOZXawA-SM&>eQ*=hXd$lB+~^Rox8VxM|{wz_~S0; z-}I|2SKY%u=J&Lp6;=g*9J+dkqZWUKB#VO~Fx2S+yOX~hywPuBLRnXrbl-`d4)305 zdl@R+-qz-0CE+=wty<+quLx?2l3T1k(d9|y??0K6SsL`jEB09V?RzITv}35_ z=b%8Y4t2Ex*1iwH(e<7rR-C=E6f*UVZlXJ)?#S7*vO%?EN`d@eILcGNi?>Pu2O~xE z#NYrm=N8eR=pl@U%y|usUlj`PJDr*Nbwm| zig9#1GqlS;l(jqwwKSHRn;4g4G{I-M46N%Y-3ElDU7Qi&Vi5vQ3nQlK-o{iKZ&M+Q zj6{!4t>Cm$glViOjjC->;PVf25Ph{NuI`mgcwb!0DAA;7aaKXMn0fk??w38i8()>2 z{t8{WNVvN?wMjRHb!hOb^EH~<>H#%g+1>Vb@zvKejv{sQN1Ao{Z5!f`TGbC{b zi?ZYN+kL*e&cjcZEiJr>ofJ(1_jQjbkck#19z|*8wIhWOKhA$vd1sGDboC#WTK@2f zaZ=FICe6pBrfdkGcxm)HTd)dOr_{^eIJFFP@VdWz=bq9_QB$L~cbvI+SmRe0E?u?M zqs!T+<$rA~47fIB!{nb8kewD}M zP)d#{8L2$Y$@MHQKu~p(%0tv=5JQjLfl#n{9?9Hu`R^hyltZ;Gg@o+fp%@Mb?mis8 z=wX~G{z`o`T1OvnVC&B(C`o~xBZCruac?z$Lx_y)GZI&uN9#u7SIB)irc@OxN(s!7 z{{8BaPq$Aa8K|LhatbmvWna%o<&$yxw zwWxarKI>3i<9Mq*6i7-VEJ+G*Jen|f3Lu_J>>?6O+0-SRJIDBvr5R7DwKU5P$YHDG zE@Inza?V|%)gdwnwi+!OH@*{fJ&yZCcDx;EWg^fQ16`@CnnJV};g1sJ zd~^lePv*VyZ@Rl8u2)+LNWW z|MpEk=X87-@VSy-lPuz#csJV4Pw3##@#@1bN+sDWqPSHre{uAu*5$czK?s%I|L*XcJbmxkI}yvA@UEB*o7k>q_HIyWD?$hm|m9NRNnN`sc^hB zbRL9n1Md`zMZi4!=&VC7zpt2S!V@W(0*{xD*31U61rfDS%F?3#oOcAtX^s7;ipTGE z)a<0(l3zgp=$Q2O52g4}VS<7q$Vdns1V?E0xai#E6=`Z<;Ag4cKiv<$vrn|wO*+v) zVy0T-r(g@1Ua)})eVRTfRnxJ`in+KB_OQ*@rOW35a4Rsh*|C+unNrKznW8HkxN}~Z{f6ml~F5%xEGk*x02_s z=@S%Fh*{R67c9H_KtU6L6~EgbJtj|Py0E)aeOlxPTian6Se$j~PE_s}P@E;bpR-oKM<;DQGLQ1=YUn*MKgk7$Z=YDJmOu+j=pplNPWHZ7Hf0U0J&U zn>IZ=vls0bUFD-j>0Q!#?IlnM@)xy<2eF-V%w}(eJo{;JpG`>UFD?>$D1%xq`!{z_ zR%Nr4r=f%ZK&eZAomkSuB%4M~uyD$7=^W^1$%lg-``6e)=n*?u5zi^=7;HQ{=hdij zV=rn;4a2q57;eye!QFVN_m0jGYqj!eNaZDo@SLyLY}TlIwog&_cx`yY-f0OP=GT)d zQPm+FXa!`~6gwz&aQRzzxB%zWVs!jeS8HEPDpg9O45KjgyBAHwD?M}HvUBHy9n4+~AMn91a*= zm)gk8D3IK-D3Y6=7!CMlq8U#u()9c#Y58a8})9gov_e4%zVg) zq7EDg!1HlM1M8jmY|!Nor$ePehvnLM_${v&T!4_0V-?n&uRMK_>ku`juO?JwXJ-cv zZU;2acK@}tCsvq}e4vz*ZxcPavdd4*#Z~O91LwT5s&QU=+#co^=!!{UG$|9znL!m+ zay9}$(4mWZs)Y&epglKqOB7vW_4Qgp@|7QMU$oEbOih_BbT=Qun!b5eNGt&e}~H{l!)#t!YA|j5{cDRK+jyxg=-MHT#rU3NAFwlq_WJfauBrAz|4>$ZzIF8DC8-V0!g;xrdy4#X%-0x{Yum%ts@P5zmnxM;}|n|GUcf}h3*+M`d={4eDY&s&x7 z_{-0<^O2qFFPuKZryRRYvJR#N*Vg*c+J_VB?+a@|;`d6e1~RA4;c(_D2H?l}3ola7 zmVqZ*_zqKJ>K-og>(FSjhlg@4HTe!sP4~TBX4b*Dd>@eJ(J9mGo!JLmZ$A7zt99E? z;-t>TwG^@4FjKAF4?5$DqEQc!V*a?36Fix8kY1~(cLg{U)i9LTH_TtSq+-RU-{0(O zC>bT~9y}B;8LQd6SuS3cSK|EqB)WseFAXydD)TToO2j!MKQ`Q5 zhN=2pv%Y_Jjp;!X_(b?~5_1+F{;5uR~;sahh1KlQH(!r*KMCP?=*k#xCewb+I!X930>*+JfdXF7aJ~gmXrse=-!b zGryoG7;{quSBCkYnt`rG!+52zZQ^c*{4ET#`4?;{KDM9&=!SO)m!F|{s*}B=HIqJAO5%a$aUtGopMX8 z4JORd9CU5JeujGo!;JN8eQG;#zs~Iw^}d; zf|vY`)~#EY+`HPDu{2jSlLx52kcHpoqw@OAIaquC)z`1)>$`8<&$nvOd^`+0svxGH ziN^)vLPLJ-nvI54HR0~cI1>hsOxXy{=W_iGv3-hWZT{w$xN3{)@?q;-`uQFF2>r{U zP4ynq$yTPMkAO`_9L~J^&`|ItIi)=y}{)JUG}B)1-~z^c)Rx3 zZMMc`mJArqCw(egg>%_kUb^}J1;k;gS zFg(583{QKe9H?{X^y`BUT})rFP@g$aSDJXfBKXdNG5z}W8(8bLPZ}x@Fy=jbrFMJ$ z<5s@Ldwh+nGxo)0&X%Qtem392y|e0%-~8WOHzmu~VIw44Dp=j-oql((V|YHiouLp{ zsmx2?SUH{SLrph+tcKyfvM99*ABtGDY87v(v39aco%uC;JiYrKhSKG$L1DpCdMUQu9YJzCtkn@&5aCn<^2hu zJ=8E^>Z}q><061G3pPHx@r2o3+i3sw_t!i?z^8P};CGbPQL3Np&?CUXOW}@}x-%2#X*dc3d}Yk&us? zI}IORo;TTh_G~u}R9XdPH^;tY7`#e^8fw=7avQ3RowS=pco1a7=Tj9oJ25un@~oP* z2UcJ9^lOYekt`Kob_IXmF6l9vh7Fi=Wjq2taDzHb|2Yt5MI#e=pR7n}XBAVuCR8cD zoMLB35zQNewfR8HMfB@CuA5qcbn9%?!3}OMgQO^X6_+?6qXzB@F|(#CTT_f{WB& z_Q&SUQPQMh+NdFlLOqeSTxOI9Jtv#J&j~dVlv1i~Wb9IuHERh`iF*D^$wN)1)wKUu zmW3zt-)C@LxuobvHOZM zZ7PS$>w{MEx2%INqW8g6VN=K|H_i9zblM2jz1n=m0+;5C^lBpYpUg}J<&bb8c2{vv zNVSt^BfL`WHv|W1r@H`BN|=c;)I^kVZ4R)DTrMZ(l=;_Yni#A%IPDtBhB!VMW~Rjj zvO{MvbcpI;sgbbWiNg0+PiNb9#1KGY5*DvU!$99vH6XUIayItr0a$urd1V~nYPKu8 z;(lG!|K*X@>AFH26GB*{?BCiL_x;|-Bx_4_dr)1G!Uunml8C%JEW~#wbSHd!`ohMV zb9>ejL04jgLH5Cpp&8C+wm|ApUpGzu>T2)U{_D1^yATV89O&ol>@1*f<;dN9nt3uH zL$}rIlmrP#W`dPh4)T3D^kigZ?Es_u%IvWUP{;1dfxQ%Flrv=MoraYt9ove>ofa*X zo5_|kYQ|W{kVIDOT8u5Iq~UUo$N?(5E@ziqd6CE0`2$9+Ci5Z3}XF zVT}(>f`bKiOhM+9l?4)2-4F^E`R&kh6-?wWYXXx$e>3H&Z(Ew%7_{GA9VO-yrAG_u zHNdk=KsTU^jZjocq6AE5hejP7@A2iaa8VkeJfIz|6P9>evhVFy215lDXtD0Dq!-yuhFK=SR?~?i+R(;t|yjB)lC@(b6LNZ<=y{w#s_7Hb%rvD6!McP}gY~`lBx^n$3AQ|+ZQGEGcq}`6a(E&|5 zbm(xs5cpRyK|@#}p@c)^(5-LI_kyPNr9be6Tg)n-l-ca2U#lAO)w-)(-I{O{>ctKD0eYWZem(MA`YhFzjLyV5dY^ICDzYj3Y;mYGxDy|o$B zQyU4}gDqUvhneRXPKPosm2vXUu$?e@Aae)hP|vX}!*4Xm+% zoGUcG@+V&)UysCbP`g+0H-J`f%DQ7`@GShg`rLd=4Wm|hrY}wWZCH2AFVpUPyll_X z_?w?cfe!4Aj!uD3q}1=hl!1i5H9!Dg4d*L2sav#er}jLR|YLD@)$EhW9;apy$O=;&zRAb;|_(N9l>7J86_C8WDe2YP0@T{Zl5h@$J+EsP0U9qH zW)fyW`ElTk7llmpHBKVoH-436qaglPuC@zAd?QaR(SWth*i!IX!oXiK^)zGW$*bV2 zFaUScdKO~Lim={y$)cD0yIa{GqPuy2InPYiA#2(B*jtFyrG&hpOkFC(;{0g)w& zNE#@%r&>!miO97^2C%%QU`SEdmC{LQXf<~WQA2_g z7-K~+$+INNpy^$5HUDj?$$%_1{e$1t3S76skLH$T=$-p&Au%*XH{ml+=& z-vnpFTdR9cLz1MH8kmlFPo^XYkDJx^0BKSZCC62scaP~ae}fujek4rPqtq5kr(T=V zBZiv(R;wNfp>oqqd!@(Q)DpptfROQzkdIH zTzQ_+b&efFtg>pe?)b)@?cWYmC)yT8s}NCR>bb{CSNY$|=@ zZl@lZJ^wn9*~4#he&f!MtZKgAu*Z|0zXdsmcn+N7+gBnr5V92Rxg7d^cTn6Iw&nv5 zd}8zI;CX%JD1=VRw#tb5=hO_Ye?*^jfQg>Iy*8IDBPDin38EBodAC$#go5{`B!E`v z=lFkF6&`qJ!7saZO*WTjXCbzqoKU~HN6EQrHB?}3(jT0sJbqHIR&9c~g$XC`fdgy3 z^-04oN&Acx4EO$BZ5ZjWYl9KmXUni9udJ#y&L?_bk6gp+HUCo;Iq#3Hgk1Vidq`<- z#`%pRpkp#-H*o4J$G?)=3ZIs~{rYJHBCw|eXG6i^)`m9kaCNh@7+gvfUbwgPh`Dqx z+RSQ}AH0emiqbjfV1}~a)*LnLbnE@+770@M#jE5cVqcPE7aXo<@_&vtK412QZ8x4j zOaqC*00l=*ugC94L!DtWmu z_~+ws?Fl^;XdNNgsmq9o`YNOBHAQ?-8HqCo=)Vu#TeE>)QJ9Sm87(hb?eb|MGjjv} zFyn*IN~Xj6?oSulk@DT1J~2xD{`4TEK!Hck1d;6clyw_6>?C8Ar2&X2I@mQT>-o4Z zz1wz}0V7ZpP=X^ML=S#90L&m>e(-yL;O8VF7uly(0G9L+Z-){!JUl#%CS~-B-t3c@ zLx*-ym2%KkjCC)C)YjlN>kI5G;;={^_|w}GPBk4JNq0t;`{OQ8JIt(MzCk?~Xy)L9 z^te$3aNQYrxlD(ent1;F0MaJ;A%)i5SF6p4L!V@SKCP(<0~1QqPT zGU#;rn|pzD{$*Fj*QK5p!_qvhOVAjfF5hUKh?{=?8EeR~C0+WVz9mK1!~w!*OSrD$ z63~uX9A9F+PB!5D%nxd$iLe~KjcML7j3=pJ1P8wt%_0Z{xgpv3;{eGI8 zZ{t;$64-%6wuwo&0scR#FV*RQ(uY7oY>(hWz5p!pZwzQYq?bPput=1~IaKaY@1t5X zmDEZTAYmzpicdFDK89*GD4F7x&~h^)37XZVT8a~o;>FoNlm^2NHW7GYnrHoZS~kj7 zKzai?dsUE5zSfa>ZZbbYgD`LRqu>`ql8?7)jWYnr%xXGtpp!DGlT#PhEFFh@+-T2F z6k!OT15Ml@TiyxIVP;zY^2^X)`n;wKRR10uBftUS{Nj0PUGP{a^3*XgQ*?nnXkK_R zB}x4-K3BV$MKToPfy$#QMroo$1T;x`*r87th*VWt0lry_?9T_!<{^-qsl~?^Sxd4Y zXFJ+jI_=4;?f5UmySwPmm4CcmqrUPCvf7qT`*+|h7u@wUt8hP)dG&I(pMNNJGUJ@L z&wg|}Fga*n^!ni7&rE^GXP^xW%s`lw@ok2{5%LGIU(s_kr4`HEma7*n>rwwlRCV^8 zUmi#n5Ai!N77%_BFL@_0!)HGMY zBo%EmTLRlEqM7X-hYySjXmDZ&IAPAX2J@hdEdfG104h4^+hLdT5hf^DNP2bd;F}0<8Bx#vnKyRnuC!~1ypkdDdKf9skE>< zX)@2u|V7 zh*GaOLcYr)*w=b$`}HCeu#KS%7p(qhnZNWm8b%3_aoDFUS~M^}zsdvcp&UB^ePT9zK8dnq_1a=vyLuc$i=*N zCt=P%l&U72@1rSqYH@NQ{rxrbPf!BkIkHxX9s)HTP~x7waeMXpj%ZDUu>ne!Rrev)8U&oAKQT!~?8B z`idV90SzKb1l6g~1X593byo;)%~p&;pQZLPpK=)*{#p)tylUii-)8221g{COa&OSu qd`}F9?$utnpd0?bXscbVtT|!q!ext6N>1wznEDRvv%A+f3;zcyOLkrW diff --git a/rsts/howto/authentication/flyte_ui_flow.png b/rsts/howto/authentication/flyte_ui_flow.png deleted file mode 100644 index 7aba1d29c682dc7b54b480258db5b7db59dd3e52..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47268 zcmd?S2UL`4*DY9Tt8H$x7|>RbAc!KMBr#)wKR^h)z8}~{?Ma`O9o8_F|H z@Wy`~8@FRC*Q_zF`q8B?_3G8D#c?I6=UatNzWM#mnwA1t0tRksYR)G$Bq?w@-%%Ddn|fB_j8rqV}AV%O(v7c3NG~8*4b7X zq@C%I6swmW`q%6QV$DA8MUt-T8(eXL>ZLTd`GOX;$(6esojgKp_BfAT@Ya`#3h6(> zr=1wy;MglU&|Rz1*Vi{X(&}FsqqFT|4^K_pEB1 zzGO>@zro1&mhuGx83E5v(GP^d_1PJoimpw zvyo->m&pngynfwuDQ@}FrGYg`W@evWUaHBkPq1vrhzJko?ygCa#jUGkIt*NBDfGJT z`lr^Wfpe{)a=wu#b8dd`@Avd{Dv347_VcRT)m-eW@#mj^e*gZx42!~Ma@;;AC#OCC z{uV5*hkJ)VU5<`@dh*n%%GC2lN#@ncvfg~9STh&K1}&cKb=GcloyzQ}hzJTic)idt zfvvS7Qmtr>y5GZx;ru$uAH7mfyq?3kb?e%asu&%kl&OENE)cW&a)kr;e$b0o&AOrp zYxLPciXZVwA7XT(%&X&B!3jnsq4V@OOxnvra+|j6r5$CZ+4ja-HRVQFw-n~?m6DRW zvu^jhVJ4g5IySZlZ+@MQ-s~xF4lh0}<%zMOD#zZ;FkZFDqcZw+A!i%2?%Lm9%t~8* zvhza>C)1)P@jl0?kLQn!_LnONTh{Fn6`knaVjQQRd3#Lq_)E77CFu?Wk@R(KZN_R5 z%7=wbE1s>JDQWfPkY1LP=)um>fCiT_i(=p1C3gDRF3vXHHEJbNQyI^@iv+SO!j*z< zx5~@Qe;*hS_G?VtBX4XRUjIY9vDS5B;6dLW8&OQ^-RWoSW#I3k?K2GW8CFV4%C&`TC&xAURx!(3{as^UUtc22 z%E-vL@ag3Y=O4wpeRiDq>tat`#AYcrU0|GWRx!878w8yr3xI zYQY~W4(!X20^IUu|BL~RE@LtEPEHZ7%3lX}1xdTJ$7_33`RHWWJwJK!$l)2qvzOc7tXq6Y%50pAm3IzGRPIQZs) zz5^3m+rv|$Yb5v1IxXdFaf^@7_RIMQ?eyDiQg)~^N@KU5)Z6h(Yy*?Bna^a)L**)y z%rw#`wj8;wm~(R_Cug>Q=&?s9hs2c48#1bgpVVd8tBBk7cyesnA$9)i?fTAR^+ES3 zft}t;@%vu47q`-Z?%I?Gp6lNPvq$&lI*{B-G8#qw5tqQacy3 zOV~9Estmn%{9}ouzJv16-u|JX-R!M}Ub4kL!XBO(zZVN+kF!oxIjuNT=*djrAJBbh zyWN=eXmb3ALQ|hvT&A~Sc0xiz{a{5VPn24eVgKSxtEOv<*YDZHlR9+b=TYOnFF7pMY9l7w zUf*Tgzn1tnZr`)3^X;QuxlQF^3Y`&Qerc_J_wU~y;@_p>624_DxnDL!KPic$(+GzP<0co@4J&-T5QuG=q*T^4hMa zjb)~B=+G4`gRqoF7oFNtZoEi zXj}HwM4Y^z1P`<0ec)xidq(OVYWBtX{!&xU?mUVCZh9khL%7#+NfDbD4@NVjpPw#R zc52|bZ*l02{jP#NHVL^bFY7X>U!UvftwZXU-qxk zxpC`O!n#2xBZa~jFJ3H+P`Rk~?D+n4zo4Z{m$tJ%aBxI~>>do!e|QEDcJt01VVB;$ zd-t+}uPs_9!;*g@7NQa+Z&KnXnJY@bYf}PYGPm_Xjzc|#PsCG#n^Tu-#Om@ed9~t$ z>%Xjy2YUMB&J?T3f6`d(g|MLBHe<5Z=fBdpW5cM#@H8?&5Q52a}S z7fsJUKk`+Tol}aw6|#^zh01n!smUFyirOk7%Jo{gk7a($^AR@lym#+-N+WLSZ1zNQ z-*`+)L)qlSSh?r_UlHa8_&e8MQ9Q z%&F1AZZiq}BvUVd3ZoR){p%U`5~gR$5-J0$gySrjRA1jTH7oT{u4WM9$VuFuYS&U*D^By<*5|w7=c7IzGf+Fal@Fq>-d)XtH+gxw2wVf*coDb@}_4i zFo4Jbx#<~SNf35l@*gW!oR-LPnMm6z=i}L9g@x|WliGT_RV`BW{(}cqdPUA7Ef2b{ zE$gjLkVcp<|K3tWsf=0hqZvC;2ASZ2r^J;3pvTv22#z-@#bLlF~ zi13MjezUi;18n3y+mKP@Oq7IVN_Y?Oy%+R_Xf=FA}a_zpnT2k)QZ=$*|wFDkIg( z{aq&`42yku46>Z8ZEVy4M};v0K!nFITd78=?Ck6{i6$QU_AM)$_7e8EvsSnayPdsH zO)Ph!!8JBERy*xNg!gv6ND+&gaD+^2D=Q^LyUGX^uB%tCw)>vVO6`0<_VnQ1Wh$VM29$+A^d;HqPza|-H%CZ12$y^YZy3t?Vxfs|9Jm2_-;TRio&Rvpn$4LFmgxGoIBK2eZKK?%JfBU zk1z?Uv#TT+7MEcK+-}7N+GY55(=p^?#(K9uOAvTy+rf@7+DE?l78n!L;`}DV8Qp zxp&fBCLE5eUbN}fNSy_d?yW<`| ze%P@p|J~!qk3Dyv)%OxG&|ke>cZY*3Mu%^&>rBz*{!$y<2bbqR5)7Jn`)x{9KrfAh zi30F#+tV2mb5JI6Fw^Y%POhrrXl0s6MQOzO*D*} zM%U)vMligJl$O9x6TyxxOMB+f9u=0JkI~(nEoe` zNSpE;Tf8)ykz%i0xe^^)c&c@Dppz=)+OyeDD%ED(le@h^TwPyZzb5{Sthz=^7E=E< z0RaKjU7rJshI?J7qJRO+Ub@X-)7-^47mXxTR8%C3a$8GpWwYXk=vaoJx}Dt%F`JIl z5+z>S9w1kQ;wz#IvR%M%6#Y{(!|fD+ZaZolljJ2E_v+4=F=L6Y{;^}XQJsf_b1{7% zuz!hO9H2VCR{SBNaqe!Eip{Y9QCL-_fofRe)nBtWrs*=y1-nlE@YhbiXyMb1I9w6t z?=q@*<|4{hR{!fIQWDDb|NQe0Xi1W8s za|v0@U%?&VFwnuXb?cGr$?}h~T{X8N*wu zOc3yS)uSJR;fO^JY}l=ssLAKHP%a;dH zq%ZR3Un}X5sGV$Xk`{g2)VJ^S=0^pA5)u*>pg`z(J-8IqQ4HIBc{A_S)V8>cr!Ett zCb+I?ZF0=`!~`ejP~js~^(GSAhLk{kFODS zeP4W}T(kN{tWn&@Yx0eLxg~G`1=od=Oh8|ZQ)Qxw3Mh`xu@>i1DbrW2;@8f{x&kqn zZd7}lYxSj*i@iXf`0hU2fogrUFK^?@)vLp>)vGs9^@Astk@FQ%1K2zR=z~>diZUot zD+kQ zqvD4<56V8+?&a_AkJuNLpMO*@!%m5HVW2~exK3jGh%g%&9ySJtdGT{F2Yo+Pjd*;j zqoo_g7^1oW>|Ro?E)%%f++^$!ZbTg{G(|kr9Rc}{4>3_EUSBIHDNz6ebb0QQ58yUd z{P9Ns)~`iFh8Cz!*&(s)@$a`dO!DvjdGWRHHr4Rs{l7P+*+lg%F;-N(2jCkFvPBtN zH+kDOY&1~mrIfTmFM{kn0743@|M21ViJ?Y~)2D;6`KXGJPo%QjisuDQ68*Q&S@_(^N}eTe8tIae#5}>FEL%t?r9m9|JK) zOqt2Q959vN2M4{w!oo_oq#wGmj7Zi7hfaQGL3jCZU(+3X-7QQM(OMxzl-o9aUk-OyC**zf|IXPTqmu%zb(>zciyrEEITd*s-wxy{+|4$i6iLTH zRqS`Mi(TR1NZP9mTr0tmx|CF9IgecgJI2JXd%L^4V{^o|mxoDJIRD1jr5>Zr$uqLR zA4qFWEgq5|MI*)pPcLWkjv(yyjV|HH{AP78tf=edy3n1bz{cD_d9uL4tvWzB`1%`EWHlx3lSLDmv& z(pkcszj*P7fzGOx%a(0kvSbOf4E4WezZ$mDz1|KRSRM-q>e)4b?&RRS^&V?bCjg)N+_F4~`_`(gV1N2I%xefFylr_!0 z&4wgaO1bCm2JsH!p7C~BOx2Zn%bP!c4#d}>GK|DnnU+4i>H-of-ErY2qfUhVzK;b3 zOaNrLp%>sS+Nk~n=%@fG3Ifj{&3P<&=jk*AOP@VfPf^m}5g9<%oIh`#67EgNy7?$m z87pxBk(ftFNT_)AXI3Iqj?@<~+HzJKR3N$ugDMAotSoHy!IPt@=pl%;NF;Aou%DzO zQkR}dp^S-&2a82o280G44vs@TJw1`?F+QV1y>Vc@iTNJNkOoHLg+gRmpB4ht)QQ!b z)o3x>?Yg(1!!zInlR;8iKyP}1=khXaq%sX_x+BPauQJMg4xajCE!LccYXXKE97~3$ z#IcLEX~b4Q{^+j#0-* zDJ)pMy#lI?#cJ-*o|cD#Vbu^TNP410yiGk?5%qs%sNA~D!5TAbTidEjv*vn2tyyj0 zRB9{@nJ)DR&4NzFkAK z5>2h^PEyp%MO2O zWa^}|jrxtwLr;hoM*fdLOm*mp2m(f9lUzfg=kEI%X^uACYvX4H1XbtFYpOSc} zAy|z`%@{Ax_>b&OLhD4Of`Z=7U?eDKUD=C=_fgt{8BY}7nuD#R3YP!yp+jxyy;sX_#~(BGzXYVr`zh;RLYtMeleY_;Q0-$0y#G%1+g3EfN8 z&(AMc0Rtg8Tsh0GEM#?nP22nZD2f$P*Pv(tDmU$t9bs(|6RY{?U1r!_pI#MPjEYzl zTd&;HUr@sS%bOcFZ{8G4-Y9nAD3^ktXH-;F;RAsEks)3ubHBdk{~G~G@nRuFuUpL@I=f;214P* zxLr2xOlURzH5JgXkuFRC62V)f)$<1NSeT;Dd(I!?3?|`0p(oF&st>_cA{1Pezt`0? z<+)d3pHf+{b_@@Opa{Btdo};vR1d@MxLOM+6082eqKUS z67iW2Y?`#x__xddynf9Y(kS8&_zc1CKJ><{@$a(4`(Yfd`-={U9A1*Mz5_~;59$`n z#;llc-_Fr}fWdx`)PJGs{P_5|DwsijgDhQ$`Mk=(f4hvfa{&}p7n^7zks~(CL!uE< zPj|$6U$Jc2Q-J+bv5(%oIkaf~9z}%2?>~O10Df=YvgMAwr9*#PAQ*RJ)DL3T&3{v{ zKp2a%>HLrzS|6N(Z54)MPrQATveY zyY~lCo@@7<3qWxm24<5~6@0a-&yD0>&NEAAy1KgBJ2-Rz*m&lgR01QUmT0V$IXT)% znbVzJQZ-7QpIB%p9!Ji~Lt7CQ7pEwjYZ~2~IphbLe8WIbz0Rdem#nR=%TQ?vxs2PL zdZ&Y>QDI?<7!K|Y$H38t{ z)Y}+mquQ?CUV*5}$iM&oJw`8G9W^jT%Y?SdFILjDoIXB3HHzFQ1d6%4E>#7$N5HIn zB6s&y*g7T$;#_%l?4Wd2j^s}z+^P8X6+GMc8#C<7AHE;Bx>{cm;^MO9%kPXVfAZu> zWthT7)cvxMGOM5ng=3c5!EAVXN<2O@*i{Xp= zxW;x%Ii!cVg;QXTu=&Rwz%dHsY!I=mN>-E?w zSLQZjmTJJh8I-U;fBuZc<(2EcQGgiJNXSvde!BagrOYqTnR^32-@W=Yxuu(S|Z!CHlyVfzK~ zNDWBj>Xj>P00_z`T6@J-v$=7}0WUHpgGK(lb~~pl)7i*zr_=B`TB!JN8E^zA zXNtnqc&ill6+a{`HL%k6Qc`Sv^sfd622Nwa6x;bxl|*CIzpy%p7@~$^itG_6q)f+$ z`)=O7s}3Dwi{%W)=|?kW%`tAyzt5|a98D1rX_Uw=`I+OA3H&Wgtl(41{{Dgot!eLYZSpsfv z9vmu&uY?2;smifu64JzwhL`^VQPqzdstb{8>Kd~c*0-?>d8490c?+`LiYMwDum9&5 z%rkZY#!1@KOzM4vPv&@x_{^ad`PIf!8@O8hwsLTswvuD)nb{rIbWY*YWB!(e`&%YI z?2=IAg9Ogej&Wks6LQOYiT%dG1oYO@^>}<84DqkuhpR>Mmf6ggOL4jDCE(SubJvrfjm z%2*#8YnM3r=I^C!S^;oV0RRPn@^}7eh<~2(s`ajwdGLd23auXYu13|Rq-4h*o`D3dWpVq+7F4Nt5;1VdEOo5Ek z@WIzL3b%?46V6-+L<}o~|LJzD7sL84kS0NLXj8=e z>oqn%Cv*S){WK`~`o>Z-K79ry{_=I;#GxZ6PO|-69E|_}*d^(K|GP@izxYJ&1zov* zJqqL)5^ja0`&hGw0s!&DW07QYVK7ADNp&-C0dS903A+pXhe^r=g*OTo*tu8?`os!L z7MXya1+}n+8DDK4sgn}>x?>^z{yzDW#U1k*FE4Wgv`{_x?AZ>G#_v9|uUZv~@ZAPq z3@llnoQwLte-~s2rz8usvN%G3ewiUkD&lEz7Z;cLixvs{<>llMe@s{xi+mH37DBr$ zi#SD;euUwBGk)BKWpN9WIOeR$Q8RMUfz@XPV`oMo+&r54Yx>IhN1%J37Drb8fV^YV zl)H#Y0wb08V822|FT!p%nmsipH8DBX>i`#+TJX`O=@YOLDS`RnAPzkD)by3^k#nY6 z{>Cy+Dl0AOUMR5=A&m)}lyTIbf`=y(kQ_V@0q#P?bwl;(fpuGdec#f3$rZuO<85tqAi0Uru)_4303MCiZx1oxM zjc^vfUYaUl0Ma3Z^5A8Y)>uuq=eGVbeUEC|1rxFWO-&AEx3soC=-CMX1|~-x4oQ?< zP``F^a&QET#wrFL052*A3KEut^?Ekbj~FjZkLV2+k{B&8PGRYd;5l#!@*IFB2tiy4 z2T^8n7x2m7VX!L#wMr!LG8V(#SOjvss=ae^UEkRqLYd{sF?;sxGl-*`91-5!t|R_1 zmLN*9UHw;e_T?qS6o|O=>uw7VL!@_`n%O`U~J_kTcSPHzd3^#LkxXs*-9Z#q}olxBtL(b|=wF!ncv+wO9tqFWX zafd167t)ry+24AHePNf{fdXKtr5Er{?}iB z#iy-heEW!1%<`<;r8H`wkb{aU662|*KE0UjG)eQ`v>bB-V=Ka&7ED@72*wF%GIX(T zifw_vGxlw5uTJ3F{~tHYm>Z>=8h_;Wnm}wjv8bq-^jY`YWSX1%RAc&ckFSa1G0#!n z#Y=P%*%+o@|MOkspXLcU^cw@=@s!7{1Eq}J&nyGrj0Jc$bl8Bq{ANsZ#E5?cjRANy zE4~&gb8~NR9Ia$6)ou8Q1kbOJ07>%j@g18tZ{9SG-3AMVmgc{F0>S`+Kd|#6$)wB> zF0Ho$7I7HsPHAas3c|0LVGp%?!;2+1`?ueWQBAOdXU$z44){f=o{DI)$T~KlE+pU% z?Z}jrBG<``Wy_Wcoo$dH*D*>VB-mQLlDUX+gNMjJa<3Sw8kz49qC7YtWOTq)Nuaad z3Q@8WB|BAE5!fc=6*m{r@rmC%IeZGcIs$(4G&p0Y3zfFggW%Fo(!}(`ZxkHynv5c* z72KqBA>4kz81bSWLy5Bu3??YxByu#Wg)8nS6KInqbvFoZpf#{hq#yRHjyE@!r`i7q z1svcqFff32UbCSbCTS|zpgd?iKbZreJbTbnGmoJ(F691|6}Ol6di^^*u5JRpogMr)87IM}5H zXO{vvV!^whUsRwXkWy)}T0ozM3gj$M2ThJzjJ6uGSyK8|d||v4d7RwL3mY2^s1gK~ z(LyYN0L0GTtg+aw3VzWj2W4b>?mVmkQ%_|(l_MY{R7pVr4w?3KfrXHa>qc%s01vIs zvZxh7O<;^f!>by84>8fDDs>$e@jc0%&A=R+e0i0gUYWm&FPuuEYFi{1QYT?SV69lY z*5~U(kP%+|I_h*s@QKFZqLEkuH83Yac#8$ zZP(8T2l9zRa3SChDW&V(`HL4t)PpWwyr>Q{(viCPc>2?ue?GlA6a}hseY%NJfVn3E zER_kZX}CLYt`@Mgz3rj$UHTPB?no2>B#MEmTI!4~qYVc!IGsq+!l&uwvmc)YHaHLI zIS*DFnbssmtm4z$bSq~w2!NH_bv038uN5-BcXI^~>uUzsU>O#%c_a=lSjCqgUW+k8 z@O@sNTCwq)+p6ul+|2p&=NH1*uy63duN=T&U);?lIG@}@)30CsFQ1<8g1bCA7UA#3 z0(vY2!6%qPdDPM$pjJ#r{7;L2LK}8j>AUNdAcMgf)*l^AFYoe_?hL{ zPhrpCKfs|6Fv7wA+8Z4USCxW<1BxEhRnf4pdu+mG^M0QW>27(-fE~hUVlFLl@)n=KpY8%b2MX`}FMxpAwNc8#1|b+64L=1!ZyRLml6&7d zD5OtLPO_=9<3^_={rZzs2f_#f?c}xvNJ2=N^#EjB1`r%!VAwslVXu=WJQfjH>pg4V zBV-jIQcX?tx^^eKLnZ=yXZv;m^GxfzCmoeh4_+7Ag6{^Ygo20vx*mDK-0#wY*mM~i zvy;R}G?;WJFM-w%{eJXEi4++>!E=1HbVb$gwfk(t+vtv2H*ViPW}BXK68;7Y5nrqX zve|vjlDv64d>yXYHQkkU`Wf<@K9nUEV&QXs$c4Cza+2>s6O+MVY^b*i?MNI<1*wTd z==EThvZT`n!A4Yo>Y=QK%;F1Fb*9R(4-u034`7#wAUFO-m*GGFGY%$F)CUkFw@<;g z5)B22$)Z?8ML+V38f+H>u*cN)_@F*i2Y*_T?dpo4!Zt<$=H<(Ml%}BcCU<&?B0m@V zOBvup%FttB(>S&J%BY9J-uuwnot&MKCM1cgWWn2Aa7Zo;*iRbdiH zvTl)?HcDh6ikcwrjJ7;5xPVq08cS9%s7KcXKQ0vCqXZ&S(HquL9_Egr z^A5HBimn{g5Msn60_T^4U*}{3E0o{gBAs~r)y!f)NuBJ8L0^;!hF!Jbs1F=CAcTaG z-bkTJa1QOneRD>?7YS6ta)Rvn*3uEFjP!~WeoAF`?_NAAnRc<87o-N1GT3?GM6)+8+kX2ErG~d^! z^;;GXAdEk)Z$b~)CMc+kaBec#f(|C=6&Gtknejj)DsRz=DcD%w!^b5eGU#_OBt|_7 zoHaj_2(7Kh9tME2SH+&Lk>#7PD~Z$ zGHk~n@S&(=$hixX0|^rt3b_Zp&{xo#1$L#bCj<04=D9|d4NlVc*olcqupqQCip6b8 zZC}2e+`WE8o@a)iR)~yO*`r-Xw_8Ch!x#RTa%IOjxZOMJMM-?zwaT;rZbcMOLZCIl z{Dq2gVOlW7K6uW7eKeYG_S5}!`I`uxA^auM3Ikn_hpWij=o$@LFC0Aj_ix|cW3Q~- zxG@gj2wGe_^mWpTU#zVSI+4hVfP&%qf}OpsAbM$8K$ajH6X}(0tk9dEoG;|!fkAW~4$^C0mTY38@+lUo5XhD-!BRiW}v2u8vT(Wt6tRP4j4p;09yxmWx`TL5~& zu=!+ByVYd7X2TZG!2~EOg;M9~=@|@o0s+t`Z53wYZ4l*~eOpH$lMO(2g1(_ehJE)#+^_$+oD7jH#wLzV%BK;JCf}Ed(7Pi4owd-(G=uwz! zaQY!fJ0BEzDb^z=6D}+w7O?*D77wKYeBI7>z>-;h&VyPIb7dfem19tyO3aALM@TE9 zAqJc#zIt@DGwx1c?a;Dy>!N^jAqPd^NplA|Wxc~;I3e3MXi{6;F72byPnVZa)x3yJ z@Q}ar;3INVL*6W^t*xzaCLCkG%D33Cs6%FxqYic@2M;$lH<&#}^@46LhQ#_B1%}<8 zME`tl%>K8ju6h$qwU)`LzLGhrlZ$Ws^@R}{vPa#oMs!{JmZ!nXJr|5A8 z?bWvHzSY^}97kIi?2jo$UQcDzJjvjV@w!R0Yk(&LeK>f973}Pmo$B;{!0)Th%wPnp zn73fTmZON-48=%FV2bnCL+4WoJ*I@Vq!=B#gqzN0T;gSJ>_#tR5noQ`oXfAg`RNBQ z<>)t#Zz(7!AV)c;gphCX;lBVC*^`G0A%^)Jmtg3=@p&(J1~>^8}xd> z=ccHw?E``3%#JlTkJ;ZvKhBw33`Jf@|21jYJ3S`1r@t40Hql|g6p`xCoXeMnxMt!y z+?50sxMveOg+t{+3?$D;HeV+++2=3#7edS~) zEny79zoR7e|LB`0#^dMff9RV%f10MB*EhlmN>9t9joiIV6PLxRd`N;?vy1f7ci(&7aRmMqcG)chznOfelFVsrbFRGF%sU4V_Y@AB(Uiy@PB7~&owzhDPOe@A}T;ct6 zUlujJD!8Y9UvWwB&G8-K7*$vqyj{k+cSAwt5f)Y>Q43Cu=H}+$2L>)$WHjWU&SPyw z8$E!;>2m?-Gl-uw8Cw9#GRdAS~~B*5IZEq#6+RYZ~rWS++Klh?Bms| z;dCNM2vRP^-1irZgq3ndsv%}C(WXoyXv6xLHYQh*W5NaS+)qsJ2MOkRh5L;gJpB@J5B&XIi(2jA5kowe<>BE$ zk;~$s_)g3U)-o*^o1Qw7|0^*%A?O1z07Q|Il`RLB4+mG*_`Q+pf5xYVSDr=z3k!;i zD?y({R#d1GmNiV!B7J|>5{ZD4y<^9wnE)t>Z{HpZ2?;TQR7f3Epj}!(Lse8%=zK7M z4FctAI0mJ08ONdxVydO3(;o_EdT=n=*x1(YJb48OAsT(nBoC8)473Y%I3Uvip{$;l z4gQ7N#>l8ix*kmcmJ3Q36Cf0A_y=e~!W)%x^ekdGIYv=i2cX9*?EPNY3LuOA^L_6> z85t6kpdc*o&M?H@fju=CW-2oveyoW1;I8?GzIJ!VkkuLusStYJ7kF@S>OHZPKr1w9 z=5#Otc#vEDsh}~)7^tMA6sZ(+?b_lEk<`Wtn+Wk>m}`gf)@(Rd(S@uF=~5AJYb86o zzVQ^s7yat3>-@$vN~|DLl5&p)9F2m%5M|h{)dE>3V$d0=PVfa=O$gmon90si$>}bf z9?9Qs?ql0w^lm+wW=Q$P$`-cm(XEWt)4`XN6A=R?wYs{R612nc=j#LTL6KDx5n}%D zzbgWlofcZFhN4hMp9^hxfCmpaEE==)5W8?a76*y~S&9Z1FDAodkNfKzcF+ zCCzBZT0r3=ch+*QUcH)pgy@?riQ!m7E-Ivc7P^)7@R+ehsWM zq**VVa~ZQ8bDiDcC`Y+DyM+ij2tSTCM&UT6cSUn$q43Q5lW?Bg8Ddtf? zbAr2ec|Crt+r{W}c=YH|VPPS!P+3WFu^i$v)_fg*W6rP<%4Qt&z@kijzEvKL`vakQ z5KFnCrsGwW46Ox`2JL}JO^NgBo5uj*`%yGDe9Oqze%mrS z0g=IkkKAryuvkcr6{}`Gwy!`xJbez3B!B=%4Xk~uR!UG{il1pp_T)PyfVXGyN z>Eye?Y2eoKyNVxD7LXFIo074v1m`+n2aq2P?JKCK;;2o$tg<};HGd^`9|sfIUkuHX zl~qI`4EEGhb6q>F%)-L>D`r5@$oLOM%s&Poj=&*ttkLPmo#3dz+R`9tcn{9KRU&S=D#7wp{(S#DW}eDY@GN0q%p3cCz`2zo z4v}r2sUgjYRdAGr(?A5K-NFl;r{*;iIZw`EUAv_gfNgHAz)Ti z!K)KeAp18_6tm_md{0(fxS^xsn!bE_O0^LDtkkjq^(N7E%7sbZR;u*L7potZ)DZW9ayVD_SP5v&c+abXt(8 zS~97GfG94+RXU8cb*Y*JNuX$4>)$&S=XVMbgvfq~wDQ?LP>d?n#f%kdQ7?bPzUj{H6$orlOILS}gOxWf3uaE2 zLeEbghD1PJ=%d3;?gMlfz?Jlz?oVUO88@%@DloX&v!s-i*986^po>cAJ-SRz3qC4T zd-xE`6v=kA_%Y95n11x^?d`n}>%TQjkOvclocKW1mu}I`5gcbv8uTHZ6gTT0Bu_ks zD*Bn!%0r5l0f->P_1ra0q>KT^7ey?^U$m=4AOhk@81s2H^XNJ+9y}}D)#C%)X9`ar zsRuA7#?xuctt7qoS!*wgG+`)CzYv0KJfWOnP{Va(X40@EFG&t{c zrWy&y;rQs!3Ud0lqW|OYsp+!v|Hhk(lVPiMbM(T>q>>PSubr+0r!V) zMJ2AMN&3a^GVHdbQ+bP-Whg#QRN4arA~se+qwO(tOer=NDl9GSOMZu)t_0|cQs8v? zUb;lxI;u0zXgfHhA*PZKwElxVpb1jf3t7FD{^)K)^04eg)?6dm_hz|BFGZ=BnU{D< z7;b`QXl$mABY;URkAh&@2sKl1nRS5rNO2PJEvCATJZgwmOcr&XV^1@cO-&<7WrsbV z{A#1HAaFAA33#AlsLhGw8^k~4?ZgT>CX+f9Hsk8p4ejvwc{>x}#&rRGgpR*tV8>Da z98R#PMBf_B_Ce$V@NSCpk9X>>170`AwFvP>YbD&2xQ^|GZrLB#O1M!O#bplqq+Mty zeFtq(uj1nZ0`?(cmcsNI1f8!4!)7ehfqqSXL7N&h5D`;CT~o0Cn1HZH+8Mpo_y-2W z?fc)T$`uX4NHO7lb3ma(9f5et)iu4Hos_2v%td5bE|VihKp{+|!(e!NIhX+20I~O~ z%|DPG7n`^ZjYZ`orh<&-*|ke$aBz?S4cIA4BxJm%bQB!*XUhw#Ce;XKPKxGqPKC0H z%7TIWEY{Qh{tM)0rRRX|Le}DKkRXs|w6mO2;b?b!?RX5!0Ktbk(ufZBclyo@{3IkU zu0xC>D%V?&9b7PZ)@+U_ypCh;oC>hdhPHutljJMqN}T`;)T^I z$?=(u6x=d@~|4%yS#Y^y-l@RNJyxnPphO z%{ZH)S|7cHm|D`spnH<^M65Jwtd_4|d8MQbh-c?$N9&a}6k^;U&|Uzl7o83$7{^d$ zz>?46Kx6fO0M{{{9$}p8;4V3+8E8?}n2n1;D-c3EDlSHL0_{}mkTcJ{Q0#)mJ_b`R z2!rhQwrP&%1^cDVi~wB|ARmCaBsigpL|{8W(Yk8>Tg*TxH7vl#I)C9p&sWFaV_JZW zW63H5lLVa`Mz|LBR6dp*CusFX}%Rj z*xpuoXZ?1Q2eJnX_LH+s@{CWp)IwJLXjhnGz9xHrZ=(UK3D)IYC)Q9{I?P|b)@s=;qlM<4ve19wd`&}Kv-2fd}#CqwxW zng#_Z*>G6~ch)qV!o@8AQp4FvE$Xn3Qa7beijt|G&b`?TnILE(WDh6b8AjHk9`)mk z#aoc^p4~^cX_2EjgK-!d={9)9(U9}p25y^AVn0=OnItghbP5hiu_XGzzF+U}OsM8d zsD5h<232NhL*U=bc7h0!^WR$!V72ejn;|Yn-Q!os?_Chq6k4;8I!rJ=I7P?2=^eh@ z7RU&9jF(&G8v0#W^+(3WYSR`mxK(2L^Z&uGuQ(&LmQ!!TBkqaDqd8blbQ+3C=K|P( zr+;@JkKv9rr+2KGHrMHAYNOdL*>WECG&?;Q;IJu*NTR$0R=`(>Hxzk^S_ZmSpB@7s zs27W|ig63d*OhMI7xXixEnoX04n72V+)hrk>DT{KQ0bEb)iF4xlBo>U3_d2arn@rg zfTg#O1Ytv+{>l1#?&m9<$>3g1UQKCfX$tn>M6s=Yb;e(mwz+d8X#Iu_vf)Kj%h}mu z|N3ht%q}acjS&7&0HH-TOfyMES(yqYzzXeN+HcV8WUHs1C1EBWdgzrDbXpR9ZU0`9GEPtm(~+b?#5W~9!0 z+#XdQ{W$q85{wLKxIhT4*%DLov5PPTpBna28nMNB1mWzG4%|5P0BjB`7+aGN5}JWO zK-Kkn_|RB%5{4Q$9=!`rmjOvEkel6r{$+eIpZ~=8cp`e{L1&oYcuQBUG4yDtQUkZ( zBXY0dl(v6snoUG5N9@?Pb!)EmIARTs`Qc*Hm%!`9$&84IaOh5|stA!?4X|c{MYcO* zh?) zx!nndw#Xkja*c$3;HD_F9{(HTiLO?n$6(bW5{83^pI?r8u79>5Z%6w~NN8w7{3HxM zq;*oaO-l3zq$pVZbp$f|S=2)favHdiiJH$7T`zxKzD$6|qu_KzcTZ0&R8^)jcxpg0 zq`{v@JBktw?E@~GXyLZDw#}JT;lky#c=6Rn(=xOzQqLy(BX|-gP_GfVGA!|nA;_0H zm}G?f775Huwl;`O_p8mbo)}OG=VpCnI`I8Adp1d%WCNkYTan6`Wq{Gx`sB)ED&thi z3*VcWXN-)t{hu;}Q$;mI8>8+({k{#r0L*c!X78!nXmf_LPiYqWKoUoBQfD(Lf}88d zpc>-nrjYzWpbH+%9$aRL=CX2fm4rdxiT8#Bf$z@DfU%=}D2v)b|BYQ+uY(gG&KMff*%VkOSuvf!t2y6)#F$dE z7-rMfPNNKb75VfJoCpTKW3gB_*XzI~_XI^n{veDr#B9Mqqz?7c#riUn%utH4d-m)x2G{L?rAfLcH1@;)nG?$ct*{o6!*QM@sH3X{TmXoK(N76G;;VS1 z?`}F_F0d2qur3x>1&lQU(3`lp!ZufB0lvYx4dG`qQJ^e7m1(ncoiDgc>}aYwU|b+f z#i{_;f=)sWIgnYb0$eD|f=ogj7XUKAW9Bfhgt|eedJ+ySVS|Kd3U6&;NeOEH zbXq9DZEvwDB2mGhq}-0f7m9s=f{DjNbmRa#g%iJw(fUQz1p2D6*9#Ba%F!(cbyIKK z=)88fE5CNolQX#9PTBxFV)hXU*yct9V~^Pxau&}v8C)PTC3 z54A#<+H(IV_{3nG6ZrHmNQg#anqGL(2`7Yk{WSFgkU-QAQ8v5)6F`%{bX*35^UbRd zivpbBdPKH%%YG^Tb1V=4c9UBPprg{0M`guE&@0n&Hok84AKXvh=t}YS)qMH(x(#ijpSd9g6kZziMQXomG z3GNs1#BfV@+qcv1!YWnr; zKMwbfnzo+NyZ?jV+{YFV9FX(oji;R+NCE(Ys6(bJSFFef833w=|Goq6iCd477o+eV z9pGnH1s<;pUS4crfxx>riHrkr)sJ%un_{QEdC5(r5;GXbpO7;ap%&F!mxvFi`Lds+#C8X{dGRH0d%1#Mn;YS=3Y^gf^PrJ zv7Yqly}M*%ETUvMh$5nR5Cnx`gC>I?7!3k-q)^Yj9_1>(9cNe(nU8#MOZPnJQC2YG z6u<{B&^>Df8gfwJlQkPih&pQlh6lj)Hufc8!M&`14LAo%YZE8uVbq`~XjmNJ4M6rD z1z|%WttWfR1^hts-Pk_(fjTeEWxt1`2*{KL*5fD()H4+ybj&rKy8&JnjLZN+tJb?k zzyJQ?#6!DGzzh-rF-+cRgiDA@z+;vOmQhU*vTho`7rH?e*H)0|yVCLlt)#-xH33Sc zQLa9O+_nU%C-6f@{ z=eBLr&6H2-E2o+$H=*FtJ2`}}bxTJJ4daG#1}5#N7l0A4@MG%vFjCBmT9-(TD`rnk zrsMHiP@|&cD*F2$oQT6MB{dbA6g%7Q1F3#!9pLkigIA9@a70q|C(yCuaFR$HP886< zlyfrwO>>&Jd}}y#&y>m~_;T}G;1TiYxhXND`bx3EB zgmyZ=i@f1D`)c#8CwqS8Zxp%7e9hL|Aj>GhHBY_-_yNoy4r2BC_3lv{M4D|nGJwVQ70-uU|KVCjogH=jIkqpIZg+n)1UAcld4hu7_wz zjPc!9y^4*EhlvAcQmnWbipk$5=9$siOW++;k*6){=PUB!Jx0#!AugHuDkzT`GS9)z zY>;w!JbgxG$WKEj&sR(;a8Wf1P=?|WNCeesv+HI}2Q++N;SHn&GBcz8N8W0I6o7XT zxJ8%2$_J+{4vu^CAtvB4IjHExechW+>mOrq^FOO)aPhaR&AqG54HGx_%)g9(8lpCB zo~H$L7IK@%)K32r!}X+yFW5dlsU?8R$dt)e8r+=_TYmoZr75xNEw6!2VOcoaDZ&Ui z2CuRwM;00)Mf&iRyN#wR<~!U4f?`&%N|C*wc=W)?KeVUuaGyT(-J^nw!clGQ5HWP|sI=qBQ1rs$ith2QldJFypDQEkKRNGPW`1p9CbDv(G z^p`?YIx#CO&&QAXrp;H6H;>5_Lc7AA3s@F|qD9e%@~@k9CkhA8Qe#h~Ak7GzZp7W+q+X5KhOzb z(DMldkk1y*GHRtNek`g2{e;{-Auectpks2{K*y+~lGk_YJN&5F5M?LWUwNFcd5em3 z^x&_Ww;-+>|HKLvU?_^HFKk(~ew51pPDy#<``=SiS|(Vo-uMsSjQJ_>a$&SNRhA9-4A?ZV}F3&XsP!aSkP8r{zmjIC- zT;O!W1_%@({0kFc`A58M5b?=+4m9HolgsG0*GnoP{%lUth}H}wQyi9_`T#(T;{Vp% znTOS!|NZ}rZDuTEhL|j44IzyrYqHE>P8f-leFoXG6{&<6V|?u#Ew&O8#=b|wShANT z8AJ#nDvA*LJ)S4?UGD4t{eIVd-S>5U@BWx2>YVfWyg%>v>-F4T6$_u!XSkmlnEiUD}w`epDC=UZOvgu8tJsURZ~F)34w6*nNGA0`rs(QO5|% zvA1iW)BlaNx%B^IZJs$$gCQjW+7FG5&F_|a*Z=**bky{!2~houQi3hrhA7F|gr(yp z*Qa6a+TT%%_vQgz@@4L@2na4S`viXj^rI|1y7@<;zMMB3(L9X+S)dqOLDy>YN4f)I zP-ztKiU0h@Pp3h(=V2hn*4HR?wh;^2fvUAP&;85u_K6&N}u3w*cg=Ld!{Pk`mujoHR5lbpY-} zEMI*<6Kd(p7=nji^(|9kCYr2LV4bKy` z^prqGWwL*Mi#Q5Bw`)ft;^N}qCWZyHeQ@DPU36B!IkTpny4D3nA&;~uiTU$OBkj!D z7f7LC-B8E>(Kc&fT-gw>z&+YzRo_CHr(&1g@9UPguU|iKv32Qj*UZ>Sg~^@ozI^(0 ztkaD9iTR?Dv3WFX-h98y50ZY>N+-lWLy0S_56_jLb_}<$=1Bk=dV+YT<1fdKTw%HB zaqN=)1k_Z@uG8M~K7t28gADES?!k?~hU46O_YM^*gX$}F*EXnWc`<=f-EfKg5YDxI z5|?+oCt@D?ai~n9Jj~lt@va&Hls6wJ{V^AdHX_7I0xE4nUH!VThm$vEwF#(KWP^%q z<7IT0zWuHpbbBV;fNF(2tL^G^$PrYg<>jk2azI{%zkt}3ygG5xq-1n`A>rX$`8AqQ zLrS@raoQS1y(;gBQ)^bjIevm_aSfp*6>r=IHIuGSLOH%5w5GzpOPKw+lCBU)D_C@>C>kVwJ1SuD{b=o?HN%MxP^VEA#5|wV%xhLp!i)Ts}G+Z zNQ3A=onYo(=$q6t{zq-kn$#=Z+XAIV;q`blsNijwVifv(d7|?E-*2I6P@Ed;M&}my zjVjN|_z#WI{#xrS<;ei_W#{P(yR(?aS4oAi*b?GS#v$qe%a5}x?qUD{&XRTO)>ZSr zGow};A?50>;q^res7i(Fc1QF!Hvi4-O=`x+rQUvVfagsfEoqSXpmJiEln$xnC~1(X zY;}ogP5UOGG6|P(OFnCa&SD_>)m$Vp062?(2bFQ|-d#h2^1^(#w^_$r2AzcvWVTIz zd3obe5AFoU%HCAq%8bvveVko+N6nenZ#>AJzVdIXS-H?RW6B$>`@Q~9>_Z|TyP#j5 z*lSeLrT>x}1zVUa)1wb5WX1-5Rh2#HPwMrL0wVs;`2f^3D7?9BH+akdW4S=jUjYv~ z1^eH@mvZ-j8eE*vRDWf~JK?JksQE9UOHhZV>XFOvs~5<%W=5FG7ydezl1i;aZ?$i1 zQO~NPl+9l|D))@=K81l|Je9I$bL?LjaAvtLbuLz}J5_L$Izp{MFhRJpM6T&TO_YztDg8n-_refa)F0eY~TDv^!a_xRL0H9>3;|H4HzSTY$wGxccJ zpn=~4G=INs_yPsl-q`baT&x_X(=X5H*vDY-#5z3}`*saf9R2nWF3(RdT(~ea5&TiA zR1Wttv04%1zg6cJ{5eJUIicgi{MK3P_OvxC zy!<{@L+N>p?H3JreIufz|4_RD{!@FY>$Pp)OG~R`9653% zxiD^+ETNCLD3((7mj4&nzlW2>(&62u(O^>I!dJiHKd{$l-)Cxf-$2hn;Rqv98p#Ex(>V@c;~ORN;1jYvq+AvQ_> zE3HI#T7|cq&`D~SCGSCb-5}2qB&c$S^>HV-zy-KKJk!ymM+d+B)+pT%xMQg?vDb6X zoj>1;&xE9?(M<$5y-?MNAF{K%)Wr0>6QG}5(+Xt1e6{}lplX{$a7IqK^X*h_FnK!L zWu8%SGtuF`YFv6Z(iIwN-C&I1Q(2yroiPk`z(hZA;<{W;XR=vQ5Oh|pw&TIj{`96C( zxA~bn!xB+0W%h)-Bbj$fnucO4)DB{`Bh93Zhy1KdsFIln%I7P!9 z_^@!_$|O|nV|1mL11|E~bQ3X~I7xt+U5<%f^Dgkw4c3Fq8(iP6UAwNyD|h4i^^y2s z!Cia9QA?n=e#3??CNt@NnU?{0hZn!EB@sMr_SzqXQ;Lb@?-wq)<*z&7nZe&n!W!&1hg zuZw?KZrn8_No<~{EUlZN2%`!;knwg7nHPFc^t6MDFHKCpm5>_Qer6IPI&~N04n6-( zay~&Wt_hWw3a0rB36|vbB_u+XQ@&WKi0_uJ6T1_#sos(%ww}ZG_ z_*F7YAN*z2s{K>yu$Wt$hlCLzwVX&+2^9bKx4+%9Ep9NpGp8$^v})QkDQ00ALyaRm zahq5{m@}oL=v=tSY z@#KIK2FK7xs}18RCWe`Oe(7j$Z;$Rw>YOKJ^w_zVFdSVvuaeo%%6jV>U%I2kT*_G{ zUTlatN8X2XW|=eaoLzt=yY){C=C;9qEH zXu*grY#0sLSgG-6_4;OdJ#+FBAm3po@czIlJgZ6Q^5uQj%+J|4`C?j^AAVRobLR1> zv*&PU-wbSyT&Z{O-er4zYOwgVOJOBn@TXWmFhtq{&fM4cN=>DWGZ%2e>Igw{N71mC z(TZu)lurPX#g4iLiXIX3`PV6OjLKjI#{|%@5^izKl(E!W$`)4KlxRq@v@hz>POOg8 zo3lNH0(4b)xI()Dq@7I~a-#q@st?8aaAKEoj7HL|jS&NqCa2mCI&5Q991-!m8YhA) z_}J?hD|R|B^|GWHFn=qW=_WTa>0%Ohf{&H5$gX|nZTKD9aSKUKatQa+AEHIRGL;`UrVgK+*ytVM|B&ArPv)>EW z8XFOdrid-O2Zd-)kRxe@Xv-^|YDs>Vvc05Qhqk56FMhb%^ixF(SLvPVIoa9Vnwk2| zqKaiLWv>bbuYknS$HJuvUNJ4=`U3Foo}k$5@xpr2KVL&JDoUSE+YRZZdD4V^w$(_X zv-RlD8sD*uWWFYawR7s-Ijnd@Mpn1^0ER1G5^!mB*pzX zyf2D`Molbb%B{54C6BH%r`pkS4PqBxu=|75_qY{0FG@zZmW%0c1VrPmefyI6G*ZyY zyoxc}1vZ)d^wBA&kgu_t-N=WL^Olxp$Wd6D=G+?>Y>&b(Ny^Vji9Ep~13EOBN2P+I z1qYY^KK{fBO?fn#2W@~{HRu^VUjfC0_N1*?q*8?1B?SFdy5)BrI><#qMRoevu@0(& zmN)T~oDDKWmZV*K_Q-(6&N6>@JwedoA#ls7aKx%HxM~jPyh>0Z%`^X~f&5Hw5n$@n zfW$oXaSw2?Yn}4jdE7K)0yvz*D)aHgo^^yxo$doXt2x5M_N2jwD7oDQou-MIwTGi#}%}BH{ zi>6LU!PC&igaxIJk$F?A>7=4DzX*9X1*iMmzS+ma!y4_giS^hBZe30Y9vo>FUgBkQ zHfg*8Er~jYnC&4Ux^yp+{P7gBy6dNAeBZeha-6^7rcUB$#6aw@L-3`RTkcDYJo&VAT7+cyu?D{ieRXzU8(gwxYMRvP8jdoE)BXwjwJT?AEf&}LwA3xDHwAU+MJL;Cl%<>$Gfao+v%Olh8503{IHX4ZX^Lk(g5aCa(Pi(YxQt)H00b@`Wzt=y=fBI z>?R~WhYyPecTDSgnsX@Cy^Urx@C1vn7tWQOoNQ8<#5XO*x5LBE?$ZnTF`~I8;|j#= zVH{C-DQ#%3ni(ov5rRP2I&kz>knHe$_h#>|W7JG14b4P-0}lRhQ!Gny*Z%$2=;wXO zZe+Jh$?-`~Sy8)IEfx4u{Lrf6LwH>Mv;PVTK~f@jZOmUS?ov`HdI^)Lcg*-$3l zJyv{a;(+!d-EcCnoD~xDCfTc4wtr#8W=ILUdCf)0HEu>lw6iv4OJMN@Yg|>qy9qpW zoG$f|#IQrS9hvr?es6r@HS}}8$#|;lNMI{kril@iCg$BE;0aiVQo>CLKHTaac3BB8 z=>zvFoQk%I7DsNL#Qj0EE|T&If&eU6zzT$}3?+Au&(46UT?9iDNiKyK61H7CP9HFg z$8yE`cjtsvbO8s9FDMM@rN*2hzfgP1(H3ntBz{@1*3&yL?epr{{?kRO zwae^qG4KX#e&wE_!-r>}Xo}x!`H7*RwE3)_Uan4y7BMkh+Sz&GF%jXi|MuiA z$~oh>_Z%{09XnJ7zme^hto!ud&v$I4o-sK8N|US=3fsagqag3FnS(J1s5e7@%rE<- z+Cl2LQ9v|!Nw)wOxTidb{B`heRKir(1L0*faF<%Yf?s`u%Tu7Z^|XdwBtV`VP}14@ zW39ZER}uI^tJ;WK`OXq_PgV5|7NY*1rLA}tL$(KSQnM1O0Zmr+_Pg)dd~T>#>A+Mw zzkT;7)S$G*TrgPT-p-oMW~womR}Z2HanuR5|v zH0mGYLhXBUKso2XT86DOObVYMWUQTfqD8A9J~SR(79a1hU&_htMkg^O>(%wShU};z zy=%>_>}S3-F179_^e+9y7q(o!$UjS4HlAMNtBwl`8y{zj4~t7=Tj{nIUK)PKPfZFh zy*}ta!h8O^-<)^PmfK!0gsbHp#;=AOTkwzO>mQ!jp?^oKyXu?PlK_d;C-?2!*E?)* zISyrXebEn$u{B1^i- z>`!09$(eZ|74KGss#T|)N-tWZ2yXQc$828Cc>z|bu_%gLU$Zcf#1xL(?1hO1SMtVi zY|=ZfM#e@`bazGk!D^LtG(+H>vH$W?yr&Ha-a!@JBtH;UZzh~ ztxd&~gy=;2i}U3v4c)tU@0P7wA!2@-Vhah4g3FZ}*2cz$no5I2wB7OJA(+FcLj(x0 zA63fBdHM1xWROlNMjg0nHw3+plmQX1B_B;a2maPxOmCdxpmCiDtuJZEAv<6MWw|;z zaHy*XeZ6FWh(8sCMN;Xm92xGwJEBB|^`tJ7j1B>m#aCa|?ANa!`bZfEO;OV2!S)>- zfA<#%y!yts;1hZtjEbJqBnpU4CHeXtY8uGv>DO{$RYyzyAXx_5P^BXD>DSK-Wwq)$ ztNN743ds1PYKt0FQP=iFjQ|;Qf}b zIAN8u0G8jIZqCZMZ{I%YM!i=8j!NE%CaDMaR+db3GD-fKnXd*?p1R8o_s36yesx#Q zUB2wfuO`q{sF~ZE4#hdiDh(>g%FKOVnuT!ZLt{|fi1CC_Xezt{y%_XkpE!XQL`e>E zk;t*Y;ZHnqzNRw8y?b#^`HW1d4zAZV*r!PQDpjh8pX6jTEm*KXiWL~j3`D!vbfTk+ z)sN%pV?}gmF^8{4Vk=2^9#JTvnbi1cq3^WWJ^E<4H6nAsXkpC z;C{D&V<%e2jiZy*0Z-1rc5X0AlTmcc5KgCeXK|!47I+<}RRxbgia3p`hvShUI*Xuu zCMj;iw-b0c4E|m|Jd}xRY0}QZ&QnRpPZ&X~3B)`S1tne&Nw6OLpMpsc zFI*^phrhoGT07p`XWVbQ=%}gZHL5_J8HXq~>P-JtJe*emDe4+2&-O?*O9a?iRFIc0 z^}u?TBxNlYNTsA;q(BYmN*^_U_sdLmk3@YON&izUG1eVD!%I1A z1+)+}x)>Y>m`#RYoT%y7ZIP{j+nDz5^_`JMGP77m<=JZk1Sl0uFKqEb`0aCS%?&7( zlsRg(iQ#3YC<>6rP-5bDxhLy*Y&LdZ?If@?%im_!OY?K)cqWDr>PQ8dsmNr*5Oyln z7<&%Yd@=>QXXE=*K1grJ36Pri8t7eRKF|1qMUp)MXNgxr6*XZuF%xWWwX(1q-?z_NF6Dbj&(pB{2fyc!$r}9+)54hm^Fb%v;osulRlr6&2Oc zFul0vIsKar;?SA0*I1C(#=|mI1>(`AiUW=q&#y+16l`^dH6V#IXUSUb0&mTkHBh|7 zr~_-xho^j)lIa1*93NY;bH|QyPjY~2{E*e-1L+Z0Ff42AE9&HK^h1jGNjelTflk0F z+b%jf8iUWzA~5aR>FpWwZ``<1d3eLy-$Jm2kcCUDmf&|-3K%->YhObyQ=+v>}gMO(!F1IdBK!D0cBQCwl9ipk)Fw!6mz#?UQCl$hw3l8Yoa;(A1Y9LOrC@Vi;`W+apfGDYm%M>tMcl%Qaj9NIzPB3;Vh9<`hWBe?AWP zyWNAM|0TjsXOl+jQluz0L=74p?RvC2M%RC7?QBy2lz3M4JUAIVxe`E>=B{v&=C}R7 zOLWYcPmIU%;nk@C(+)GFFbP2RL1F7D=e!0j@-n`uGX}HTazQjDP54L|q!Fx2Is2GUr6vI> zfeXeow$xbQ;9v_<#oQ>YdYZ*(+y-d^*aCGAo214B1*Q5V76^ZoxB&s8*tAlIo~H01 zvTriZ{9G;rCe)8lez<(NCNA=`!nmQ}&rC+%u} z_Qe-F=FiMkY#~6<3F~V=0BLlOBQ&b zoH+dcRlgzr%Yc}^{Q;=McHw1`>l?#L-Lw6~P^~rSt8FH(QUAf-JG7N^o&QwXS0eiWv+iXI7eP-IN|tj1yu2(mdne z&N06WL!B>Y)GF!raq=%|4Ux3s=09ooRSm8{%+!7A!1;0}v(%!J_%wY5JL0DEF}IzC z-!PQ?yb#Fe5C4sH)4%D}m-j*aJE=#RX&|fYmwhP16@&D9*e5P0s4Iqgd2MF5(zM@` z0}{&5`0#mf>D{~8rd5cm@+=3Cy9^m^zh-sS#O#cswn|Q z0$|7wjX9wnv}SnGJ=@JE)&v!)Q}%t%-N=9Xx1MjR7GG-KS{bV!l1x~|eOb|J)M3)H zV^fmQ0urK9X`8E`IVO-G16om!{RZulW!tzd)O~jDy?0$~JHs^l!<+H{{O?VpukR8% z8-kUSdZJ5%cOUvZ3ctrvHfbPv#a9;%)jmPLyw*ygc_;fD&wN)<%(>ay;g|GAtGwU+ zNWX0>yRK<&bWp{~k<0SWETNM7xonv`UoFnxeW7y0S-PyFa}(G8^H286yfLTarM9Ko zxbU0=TP625IIGPwmh+p|un1Rk=(?ba-*7c|U1~e;i}&TNuXER|Ds(Ft>do_G{`wz8 zH~veANxM1Mb`S5aVKcRBuZ!staNnwK-Lqhz^bg{@@7UR4p5%|cIABf=INL_i`iMlB zVi|D9PI$dGW6p*8C(aroN2v4QPcawP&vbR^(8t2>6YKso9}G5jm)Sag=25<$xkaQtg|2{ z0!mArm8=;YR8cHzkrc9zQs-f8nETq zlMDV$mYffEp|Ryj2~%4P5_6V*pN`JnDU}5xGwKcUG-QdC>GkaL5PK$%$`?+H2PTogA|7Ht!)t8g3 zt5FH`gf?k)<{ap`hr@9b46R(Hba=Lxb4uKK?iSs(5@kt)#2}vK!H1tg0@j1aqrpym z?LkS#(oj09a-=lf$j51`FCahVLLDQMel7yjh|*R%t(1|lyQQ`ar%~6P08-n|g;Gnb zDqbD4W}wI10*}S3TV3x%V`2feJ5v@{S-l#6nV4pt5 zg{C$OdjB$vIu>=^bjL-E2Im`iA&4pj zM!*vpTXy*_21hf$J>FYW;o$QiHF}ZhM2N&>Q_h}pcL?xWAb(W z0{|QPpJ?u22h7&RGgmHKW{o^uAq^}$zo!>S&eDix32YImy?k~vZ(CfY?zTlvmQ(4E zC_v7GTqw{6#G!DiUD$mABJL$t($fGOLc73W*!81Pl5M76k^r9@qH47Ps2okYD`$@{ zG!f8(vq=bD{q(Z;Qm(aOCOl~nHq4ikxbx2yg)S`AQ{R1B> z36io={S9p;(zxc0*W@yLf4@8BDw_6SCDB}5TU%Q_HR^v6q*Pj+`2NdQ5N>!lT*->l z9Kdm_QX-zP!T}{UwnR~{SaM`6lsQE>FndA>@S!V%)_bNX!mx(XRAW`hz z?Vy6G$+bE=^?%X7gL(cOebVy`85XH=&+hI2#n%L>OZW=oozzg;yRnOTcGiNa5I7ZeBi)=A_5e1k{%KG zBQLzvif;CI2puqv5DgCk+^hWUy4!4pJP?sel8i4qLI@usdyQg2p_-7>3dnc~&@f$Q zrbiAafu3Yul^3n3zUkrISvTbnUd!=tz#8yXyUq-_2a zqv}6BZEOpfjc`QCG*wA=SX5qR_tSGLy$RzSfy<^!j-n4HioC1 zT~AO6t5txga;VS5SZ?7|;7r=2h$&Tc8W>9nV>CT4pN44s*6YdC9B-0+Mqi08^K-4~ zfp7CG&qbN-m-0&Y2Ev_*s3kO&LMh*vw0*8Q4D|`TPu(`^Q2q9q>n8!{UijMjp_AW+ zm$7Jf1^$CwR`J?PF$FUrJq~B?!_O_ot$wg9Hu^e{E!ANomh&jKb!p>t_yv*1C2)uu zL;E7|t7X23EJZnh!&Dx=GBuM1L`j%YtJ-ExK&Me@__Ct@O;Qg3cJEg5jY+<~Zb*sr zW>t)`FeDtJz{reAMA9ZjF)y6H+|G8>(SZ}tEl9*J*Dzqg(6M9T(psx1)^h|Hv>y23 zOniL2WnJf%Q}@<5(B>-WBC++rQ})#|p0#;3(C1YBL5B|^#PPiB1en0vx%GVKk%G(v zQ}ZV&o0CVwl_D~;C~DY(f5H6Inj3x=*s;aDD0Y-4&6~S}x=;l4 zzEkh;{@9!MuJVR8zF{I{nC#}wnl&56a9_@l4G54m&;l^L$+W>Qa1)Q96cvxJok1aF z1o%M6G+&6?Zc0e~p7F0=rj4IqX;>7I;$YG3F4t5JB^Uq|H(0mkhXte<$F^%h5VIbb z#0k`O=l4&gsL+dJ07>x;y_Xu8`97sxjs4M@)$lO`NhUwRMqpP>-#C=2yqgnGd|Zy> za4w0@7(=xnRD^bq7j@>41}f?XC4ZGCWG|ZxVXVFR;BjeA2jZgH!vnVcv?PiO`~FR3 zQD|_>y!w;qtMkA!q(NTWz(harcRAtd*f3>*O2_0xh$ES0S-P$5*=^eS7thH zU`-v9yOe_}F2k!B!!!7>lOKe*Yc9G}rwmfCZhYdz(gh9{tG_(3I`{UouvICs_gNKJ zUVCI7nb7vJgJtQ-96!p0=NPUM6hPU#MyVYrJ*r&m9_-_E9Rzk)30QiVq0<23Y*`m_ zCWCn4J6`wv2o%ZJFD@1Jzg)p0GL8U!da)k#G$I(7eL2N8@4JQI$d=X5J?k^OZ1sdI zyU`DNjvTps+`E)>OE)0%;?zq^V|+pq;`?5Qrv#vxvMwB^wK#b97h_-{OP4s_CaeGC zSYBCv6wO9KLMF>S+wvn&@TFAqSSGys=hfvWsno*a;-*H#?tgf8YX)T=Fj~OEoL7~W z8Y21^f0+b1bnd6dnx~1d|NMloOKgpjd)n+bk6$LNVpzn$o-X4+u9rrw+Pin=FTV&9 z#I9OtHNBIW9$+_nc)+ft*R)Tpjl%4VLYurFMg~uqY0mDU-6ng)Gh!;AR-rSS_@yHL zKpN^)NLg9%Hsf18DrozYl$U@IN2b}0W!WHyS%jF5K=3+rEm?-V| z9@Djk_D_!nLlfFAMp@fILc)(we4>b+Z@Xe}v1FxM8EFibBa*E1U`rz}NIHl=3YyCM zmq@LRl4%4xoMfF5K_3DjQ8sg>q}9cG#7?@F+qHS(YH%Cr51i{{4uCgmkI8oSgEH~p3dmM=n0E_*u{Z3$J9CUGh3m#0sQ zPGMnukVhO)y(s~_n{8m1JAdJV=DDWb(lNJ*_NhBao@xT1oPDFOuceYmTd|*|u|F?~ zUIUL!R}QN%h*x4KdNHrou}j@xN&uKMq3*P6_t{y%t&jh?`u4Q^F@hX0y-EGZMA$=) zVGTHN^5u(1)XyA4Mw2DqetRpWDrmJp{bohfu+G*5lSm}$r(q@UhQH{1X!g6a(h zEG~+aRt*NCnWh}NZv3m3pYZ;?e#h^XL}fHF}PJ< z$0y6ycyy5CNT4*<+=-|*G|Yz6#x}crDn8KmA_CMTqEwS|0ZBSJT|~ z5zj$SRmNQJ_?P7`T(}@xE5a)2k(3LqWGjhdT@xPI$O5ITxk=CZN)gOvmcGc%Fl!s0 zg@9o{ba7>8ThM&{!Krl@1W&NvNGB~OlaQh~cGNmE|A=HH!D*mtcoz9MA1Dx^eK8`%^=v`7uC8oqljm=vTcp}nR zI2UoF1TuV+x4Gwrh0n*mPg$Iu6?cGjAdwlpoO;nm^hbaSsmEJ2bsTnnSE+zCi=@V@`?)q-k*JTnij;k_|j#J$|Emhd?M(&XtvnLE}+G6{n-_STnTSWnRV7q4(L(jtR zw>#tBt8IE$2b>w+QdZC|W{5deUcDG1KcAfY9&EF;NE5 z@u|k#5O@9mm5$DQ48UM#_lcF=Eq%^sTB z^H=`mcm5wVgH)ItIpXJ^d;Ixb5&vI@qIkbIJkI)l#Fdl@r6Y%~`6m=L-NO3%++D(G zJ>aB`msudC0!J7YO$P=sihZ|pcYVPRo;Y!0@U~*@dPM%y&juYSTBl!+SBV|UTJS*k zF=ECZYKC~nzs~*{+IaWEV?8k-8ZVO{I`=?1L&q5(`b-o!-l@Q*)w}m!1dr}`z3qil z)c)HVf4T=jD?AHwXe7{UQS%6dp7o}jq1rAsHkALW*D3IpZr!?--Ptw8Go;0V_cBdT z;d19gY#KH4Y`i0-QFhGQAg8Lr-q-MNy^JHM{R$p};a|Us1S7dYRHOH(I@3+Z!QNSk zu53*6VuQ=g0g>(xBHa-z+ZHT)Em*cYAFtNu788_&KpZ9#->UP57F}dls#GbSS)58r zh53K71tUKuK^Y>4{`|A(E`9yGVCw$`ukJW0dHbO4bU%0;UQ}Pexay{g)&s>}+@RqU z34CSdp^vlht8e9+&4r7NnOFD;4Vj$* zt%&n6%?SA(JZ|)|71jIv<<=m$km7@5=Sw6 z_?l%*XIodIcpO>crYKs|fod1pwVO^ktM-LYbhgk~aTSM16dTEADiqX6NvwWFnCje9 z| zBe!TprRpKp;+CbeL|MFyx-g0Us3=Y&UIs@uMF9(RgP-mt<_Y~Gr#lcn30l60#fYB2pw2m#M&2Y44 zaFCmPhE>o|F*$@m?yD)z> zhrae*f@*}X>SA#E0*;M}c&2#`dKP6s#zL^yFKeKAI3Af}Tg;j&f^VoZk(f_?58kAj z!b(~7fLlx0IAGGn@}bUHnmB}Wm?3lP&>sjgYCyk;5p^rI=W@sRwttj1`}#N+EX=_G znkr4#)eE?9exgCXIjnf|T|kYa00ndD zy%Z@V;T>&OR!o7o5$ThVtC*~iQR?{MS8F<~cPwC1eOg6sp#EP&7-Y<*_Ph){I5920 zjHa%F-iBHc0i;M8X&KCxe|9AyQ3$}|rWL#K;WTSUI3f3gI#b4M^!2s* zj_~C{Q=ZR~bB$8rM<^lA`B*%~YV*kW-^}xX@HSjAYupxLY3$FF8E&jbLEdhfg;QZ`7a~n>-3iLquT6E< zb@LCcn_KMJwaa8h#d#VSkEQGPO>O3HNS#Z$qUX?j0_KeF$rGCP;M{@jfXdyBO-h84 zch_v}YNkIO>i>oaqKjat@?UF^1Q47%N~x}WkK2KuqX^SVv%Axjei(X7tu(ckhOL^v z{DzGjxv6};4?%%`SIYU`jQ4)G*_b?k?Yy&RyyJUVH6Jx}Z7E|d&!a~>#x@%hRk2Nt zMo(s0{8{%!qugS%uGanK(&;Ug9?cki+VadC8;fI~uZ$1$zB}G8b-?TG8Kb_M(zbEw z^Q$I||1kVbT8l+F#ldmwt-KIA{88MLH>3gBHd$U8t)!uJnKB^Df3W^WdU+MCvApZF z=CRv<|9$SkgRMft!cbeT^{{zy_xPukD_1`2Rc=l(I_61@to*!_E&MX8 zLYfSnJbA^OIiH)(Sh1WNWYr5hwO6r>bND#JtU{6|x|1c88jvI*2Sh`PFo)rZYoZc;7oQ z&VOytYI}=1Mw6A%^x-EjFR!GlSD(LoGvP@_#<`z+w+bHylE|Gqm#~CiJiTu3ghTT2yAwgj>3}wF+xC!h zG56K0X@2KhZuEEj;wT|}OMZxYCv+kE+SUuavvvFThwa>HcjM;GGsAYn0JzvxTpkfo zcXeXH4q!0{cIErDnv^dg8T2S@nC)#U=AW!rHsOno=1&~e_RqI(TeWG^hCN~yio1vD z>8$i2W5+J+u;%EKCk>W&jXreP!-gvSDC(mC9*+WF>mp4Z98ep4*SYhjFvN}lFMg$Y z?3ULA^wH_XyCcr)IataAK3-w40o_uc0Rz?}AD=#VZpps=`td$3N)|_+zj4bJt7?nF zzB_f{VZ9G?7Yw=7vuDrTygdGS{KbnVNJM+Nxp_lZtlkFr@k6WdhY$!hwzjT=1|_DX zRK0cU)?&ixy>$X9c|Z-EcKY{sJb^BddQ z{lRm3J=X8roSYmkh+X5x**6!pp7{+!S9B?Y4*m99iB6q5ZQQ)M=pEKCLL48$2sx`e zwF)nBhf}FPPa2S{&;FE7v+o6cI((CFL%9<mzoSohyOQXkC>F@IoFRJT9ZSesXYRsF_A3)h2{ zaYiL;=fW#e@_f4HygCqZfWfJ`_ on the ``Config`` object in the ``auth`` package for a discussion on the settings required. +Users accessing backend services through Cli should be able to use OAuth2-Pkce flow to authenticate (in a browser) to the Idp and be issued +an access_token valid to communicate with the intended backend service on behalf of the user. -****** -CI -****** +Using FlyteAdmin's builtin Authorization Server: -If your organization does any automated registration, then you'll need to authenticate with the `basic authentication `_ flow (username and password effectively) as CI systems are generally not suitable OAuth resource owners. After retrieving an access token from the IDP, you can send it along to Flyte Admin as usual. +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0NsaTogZmx5dGVjdGwgbGlzdC1wcm9qZWN0c1xuICAgIENsaS0-PitBZG1pbjogYWRtaW4vY2xpZW50LWNvbmZpZ1xuICAgIEFkbWluLT4-LUNsaTogQ2xpZW50X2lkPTxhYmM-LCAuLi5cbiAgICBDbGktPj4rQnJvd3NlcjogL29hdXRoMi9hdXRob3JpemU_cGtjZSZjb2RlX2NoYWxsZW5nZSxjbGllbnRfaWQsc2NvcGVcbiAgICBCcm93c2VyLT4-K0FkbWluOiAvb2F1dGgyL2F1dGhvcml6ZT9wa2NlLi4uXG4gICAgQWRtaW4tPj4tQnJvd3NlcjogMzAyIGlkcC5jb20vbG9naW5cbiAgICBOb3RlIG92ZXIgQnJvd3NlcixBZG1pbjogVGhlIHByaW9yIE9wZW5JRCBDb25uZWN0IGZsb3dcbiAgICBCcm93c2VyLT4-K0FkbWluOiBhZG1pbi9sb2dnZWRfaW5cbiAgICBOb3RlIG92ZXIgQnJvd3NlcixBZG1pbjogUG90ZW50aWFsbHkgc2hvdyBjdXN0b20gY29uc2VudCBzY3JlZW5cbiAgICBBZG1pbi0-Pi1Ccm93c2VyOiBsb2NhbGhvc3QvP2F1dGhDb2RlPTxhYmM-XG4gICAgQnJvd3Nlci0-PitDbGk6IGxvY2FsaG9zdC9hdXRoQ29kZT08YWJjPlxuICAgIENsaS0-PitBZG1pbjogL3Rva2VuP2NvZGUsY29kZV92ZXJpZmllclxuICAgIEFkbWluLT4-LUNsaTogYWNjZXNzX3Rva2VuXG4gICAgQ2xpLT4-K0FkbWluOiAvcHJvamVjdHMvICsgYWNjZXNzX3Rva2VuXG4gICAgQWRtaW4tPj4tQ2xpOiBwcm9qZWN0MSwgcHJvamVjdDJcbiIsIm1lcm1haWQiOnsidGhlbWUiOiJuZXV0cmFsIn0sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :target: https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0NsaTogZmx5dGVjdGwgbGlzdC1wcm9qZWN0c1xuICAgIENsaS0-PitBZG1pbjogYWRtaW4vY2xpZW50LWNvbmZpZ1xuICAgIEFkbWluLT4-LUNsaTogQ2xpZW50X2lkPTxhYmM-LCAuLi5cbiAgICBDbGktPj4rQnJvd3NlcjogL29hdXRoMi9hdXRob3JpemU_cGtjZSZjb2RlX2NoYWxsZW5nZSxjbGllbnRfaWQsc2NvcGVcbiAgICBCcm93c2VyLT4-K0FkbWluOiAvb2F1dGgyL2F1dGhvcml6ZT9wa2NlLi4uXG4gICAgQWRtaW4tPj4tQnJvd3NlcjogMzAyIGlkcC5jb20vbG9naW5cbiAgICBOb3RlIG92ZXIgQnJvd3NlcixBZG1pbjogVGhlIHByaW9yIE9wZW5JRCBDb25uZWN0IGZsb3dcbiAgICBCcm93c2VyLT4-K0FkbWluOiBhZG1pbi9sb2dnZWRfaW5cbiAgICBOb3RlIG92ZXIgQnJvd3NlcixBZG1pbjogUG90ZW50aWFsbHkgc2hvdyBjdXN0b20gY29uc2VudCBzY3JlZW5cbiAgICBBZG1pbi0-Pi1Ccm93c2VyOiBsb2NhbGhvc3QvP2F1dGhDb2RlPTxhYmM-XG4gICAgQnJvd3Nlci0-PitDbGk6IGxvY2FsaG9zdC9hdXRoQ29kZT08YWJjPlxuICAgIENsaS0-PitBZG1pbjogL3Rva2VuP2NvZGUsY29kZV92ZXJpZmllclxuICAgIEFkbWluLT4-LUNsaTogYWNjZXNzX3Rva2VuXG4gICAgQ2xpLT4-K0FkbWluOiAvcHJvamVjdHMvICsgYWNjZXNzX3Rva2VuXG4gICAgQWRtaW4tPj4tQ2xpOiBwcm9qZWN0MSwgcHJvamVjdDJcbiIsIm1lcm1haWQiOnsidGhlbWUiOiJuZXV0cmFsIn0sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :width: 600 + :alt: CLI Authentication with Admin's own Authorization Server -Flytekit configuration variables are automatically designed to look up values from relevant environment variables. To aid with continuous integration use-cases however, Flytekit configuration can also reference other environment variables. For instance, if your CI system is not capable of setting custom environment variables like ``FLYTE_CREDENTIALS_CLIENT_SECRET`` but does set the necessary settings under a different variable, you may use ``export FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_ENV_VAR=OTHER_ENV_VARIABLE`` to redirect the lookup. A ``FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_FILE`` redirect is available as well, where the value should be the full path to the file containing the value for the configuration setting, in this case, the client secret. We found this redirect behavior necessary when setting up registration within our own CI pipelines. +Using an External Authorization Server: -The following is a listing of the Flytekit configuration values we set in CI, along with a brief explanation where relevant. +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0NsaTogZmx5dGVjdGwgbGlzdC1wcm9qZWN0c1xuICAgIENsaS0-PitBZG1pbjogYWRtaW4vY2xpZW50LWNvbmZpZ1xuICAgIEFkbWluLT4-LUNsaTogQ2xpZW50X2lkPTxhYmM-LCAuLi5cbiAgICBDbGktPj4rQnJvd3NlcjogL29hdXRoMi9hdXRob3JpemU_cGtjZSZjb2RlX2NoYWxsZW5nZSxjbGllbnRfaWQsc2NvcGVcbiAgICBCcm93c2VyLT4-K0V4dGVybmFsSWRwOiAvb2F1dGgyL2F1dGhvcml6ZT9wa2NlLi4uXG4gICAgRXh0ZXJuYWxJZHAtPj4tQnJvd3NlcjogMzAyIGlkcC5jb20vbG9naW5cbiAgICBOb3RlIG92ZXIgQnJvd3NlcixFeHRlcm5hbElkcDogVGhlIHByaW9yIE9wZW5JRCBDb25uZWN0IGZsb3dcbiAgICBCcm93c2VyLT4-K0V4dGVybmFsSWRwOiAvbG9nZ2VkX2luXG4gICAgTm90ZSBvdmVyIEJyb3dzZXIsRXh0ZXJuYWxJZHA6IFBvdGVudGlhbGx5IHNob3cgY3VzdG9tIGNvbnNlbnQgc2NyZWVuXG4gICAgRXh0ZXJuYWxJZHAtPj4tQnJvd3NlcjogbG9jYWxob3N0Lz9hdXRoQ29kZT08YWJjPlxuICAgIEJyb3dzZXItPj4rQ2xpOiBsb2NhbGhvc3QvYXV0aENvZGU9PGFiYz5cbiAgICBDbGktPj4rRXh0ZXJuYWxJZHA6IC90b2tlbj9jb2RlLGNvZGVfdmVyaWZpZXJcbiAgICBFeHRlcm5hbElkcC0-Pi1DbGk6IGFjY2Vzc190b2tlblxuICAgIENsaS0-PitBZG1pbjogL3Byb2plY3RzLyArIGFjY2Vzc190b2tlblxuICAgIEFkbWluLT4-LUNsaTogcHJvamVjdDEsIHByb2plY3QyXG4iLCJtZXJtYWlkIjp7InRoZW1lIjoibmV1dHJhbCJ9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + :target: https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0NsaTogZmx5dGVjdGwgbGlzdC1wcm9qZWN0c1xuICAgIENsaS0-PitBZG1pbjogYWRtaW4vY2xpZW50LWNvbmZpZ1xuICAgIEFkbWluLT4-LUNsaTogQ2xpZW50X2lkPTxhYmM-LCAuLi5cbiAgICBDbGktPj4rQnJvd3NlcjogL29hdXRoMi9hdXRob3JpemU_cGtjZSZjb2RlX2NoYWxsZW5nZSxjbGllbnRfaWQsc2NvcGVcbiAgICBCcm93c2VyLT4-K0V4dGVybmFsSWRwOiAvb2F1dGgyL2F1dGhvcml6ZT9wa2NlLi4uXG4gICAgRXh0ZXJuYWxJZHAtPj4tQnJvd3NlcjogMzAyIGlkcC5jb20vbG9naW5cbiAgICBOb3RlIG92ZXIgQnJvd3NlcixFeHRlcm5hbElkcDogVGhlIHByaW9yIE9wZW5JRCBDb25uZWN0IGZsb3dcbiAgICBCcm93c2VyLT4-K0V4dGVybmFsSWRwOiAvbG9nZ2VkX2luXG4gICAgTm90ZSBvdmVyIEJyb3dzZXIsRXh0ZXJuYWxJZHA6IFBvdGVudGlhbGx5IHNob3cgY3VzdG9tIGNvbnNlbnQgc2NyZWVuXG4gICAgRXh0ZXJuYWxJZHAtPj4tQnJvd3NlcjogbG9jYWxob3N0Lz9hdXRoQ29kZT08YWJjPlxuICAgIEJyb3dzZXItPj4rQ2xpOiBsb2NhbGhvc3QvYXV0aENvZGU9PGFiYz5cbiAgICBDbGktPj4rRXh0ZXJuYWxJZHA6IC90b2tlbj9jb2RlLGNvZGVfdmVyaWZpZXJcbiAgICBFeHRlcm5hbElkcC0-Pi1DbGk6IGFjY2Vzc190b2tlblxuICAgIENsaS0-PitBZG1pbjogL3Byb2plY3RzLyArIGFjY2Vzc190b2tlblxuICAgIEFkbWluLT4-LUNsaTogcHJvamVjdDEsIHByb2plY3QyXG4iLCJtZXJtYWlkIjp7InRoZW1lIjoibmV1dHJhbCJ9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + :width: 600 + :alt: CLI Authentication with an external Authorization Server -* ``FLYTE_CREDENTIALS_CLIENT_ID`` and ``FLYTE_CREDENTIALS_CLIENT_SECRET`` - When using basic authentication, this is the username and password -* ``export FLYTE_CREDENTIALS_AUTH_MODE=basic`` - This tells the SDK to use basic authentication. If not set, Flytekit will assume you want to use the standard OAuth based three-legged flow. -* ``export FLYTE_CREDENTIALS_AUTHORIZATION_METADATA_KEY=text`` - At Lyft, we set this to conform to this `header config `_ on the Admin side. -* ``export FLYTE_CREDENTIALS_SCOPE=text`` - When using basic authentication, you'll need to specify a scope to the IDP (instead of ``openid``, as that's only for OAuth). Set that here. -* ``export FLYTE_PLATFORM_AUTH=True`` - Set this to force Flytekit to use authentication, even if not required by Admin. This is useful as you're rolling out the requirement. +Identity Providers Support +========================== ++-----------------+--------+-------------+---------------------+----------+-------+----------+--------+ +| Feature | Okta | Google free | GC Identity Service | Azure AD | Auth0 | KeyCloak | Github | ++=================+========+=============+=====================+==========+=======+==========+========+ +| OpenIdConnect | Yes | Yes | Yes | Yes | Yes | Yes | No | ++-----------------+--------+-------------+---------------------+----------+-------+----------+--------+ +| Custom RP | Yes | No | Yes | Yes | ? | Yes | No | ++-----------------+--------+-------------+---------------------+----------+-------+----------+--------+ ********** References ********** -RFCs -====== This collection of RFCs may be helpful to those who wish to investigate the implementation in more depth. * `OAuth2 RFC 6749 `_ @@ -94,4 +108,3 @@ This collection of RFCs may be helpful to those who wish to investigate the impl * `PKCE RFC 7636 `_ * `JWT RFC 7519 `_ - diff --git a/rsts/howto/authentication/migration.rst b/rsts/howto/authentication/migration.rst new file mode 100644 index 0000000000..fcde70b5f9 --- /dev/null +++ b/rsts/howto/authentication/migration.rst @@ -0,0 +1,151 @@ +.. _howto_authentication_migrate: + +###################################################### +How to Migrate Your Authentication Config (pre 0.13.0) +###################################################### + +Using Okta as an example, you would have previously seen something like the following: + +On the Okta side: + +* An Application (OpenID Connect Web) for Flyte Admin itself (e.g. **0oal5rch46pVhCGF45d6**). +* An Application (OpenID Native app) for Flyte-cli/flytectl (e.g. **0oal62nxuD6OSFSRq5d6**). +* These two applications would be assigned to the relevant users. +* An Application (Web) for Flyte Propeller (e.g. **0abc5rch46pVhCGF9876**). +* These applications would either use the default Authorization server, or you would create a new one. + +On the Admin side, you would have had the following configuration: + +.. code-block:: yaml + + server: + # ... other settings + security: + secure: false + useAuth: true + allowCors: true + allowedOrigins: + - "*" + allowedHeaders: + - "Content-Type" + oauth: + baseUrl: https://dev-62129345.okta.com/oauth2/default/ + scopes: + - profile + - openid + - email + claims: + iss: https://dev-62129345.okta.com/oauth2/default + aud: 0oal5rch46pVhCGF45d6 + clientId: 0oal5rch46pVhCGF45d6 + clientSecretFile: "/Users/ytong/etc/secrets/oauth/secret" + authorizeUrl: "https://dev-62129345.okta.com/oauth2/default/v1/authorize" + tokenUrl: "https://dev-62129345.okta.com/oauth2/default/v1/token" + callbackUrl: "http://localhost:8088/callback" + cookieHashKeyFile: "/Users/ytong/etc/secrets/hashkey/hashkey" + cookieBlockKeyFile: "/Users/ytong/etc/secrets/blockkey/blockkey" + redirectUrl: "/api/v1/projects" + thirdPartyConfig: + flyteClient: + clientId: 0oal62nxuD6OSFSRq5d6 + redirectUri: http://localhost:12345/callback + +From the Flyte-cli side, these two settings were needed: + +.. code-block:: bash + + FLYTE_PLATFORM_HTTP_URL=http://localhost:8088 FLYTE_CREDENTIALS_CLIENT_ID=0oal62nxuD6OSFSRq5d6 flyte-cli ... + +**FLYTE_PLATFORM_HTTP_URL** is used because **flyte-cli** uses only gRPC to communicate with Admin. It needs to know the HTTP port (which Admin hosts on a different port because of limitations of the +grpc-gateway library). **flyte-cli** uses this setting to talk to **/.well-known/oauth-authorization-server** to retrieve information regarding the auth endpoints. Previously this redirected to the +Okta Authorization Server's metadata endpoint. With this change, Admin now hosts its own (even if still using the external Authorization Service). + +After version `0.13.0 `__ of the platform, you can still use the IdP as the Authorization Server if you so choose. That configuration would now become: + +.. code-block:: yaml + + server: + # ... other settings + security: + secure: false + useAuth: true + allowCors: true + allowedOrigins: + - "*" + allowedHeaders: + - "Content-Type" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://flyte.mycompany.com + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + userAuth: + openId: + # Put the URL of the OpenID Connect provider. + baseUrl: https://dev-62129345.okta.com/oauth2/default # Okta with a custom Authorization Server + scopes: + - profile + - openid + - offline_access # Uncomment if OIdC supports issuing refresh tokens. + # Replace with the client id created for Flyte. + clientId: 0oal5rch46pVhCGF45d6 + appAuth: + # External delegates app auth responsibilities to an external authorization server, Internal means Flyte Admin does it itself + authServerType: External + thirdPartyConfig: + flyteClient: + clientId: 0oal62nxuD6OSFSRq5d6 + redirectUri: http://localhost:12345/callback + scopes: + - all + - offline + +Specifically, + +* The original **oauth** section has been moved two levels higher into its own section and renamed **auth** but enabling/disabling of authentication remains in the old location. +* Secrets by default will now be looked up in **/etc/secrets**. Use the following command to generate them: + +.. code-block:: bash + + flyteadmin secrets init -p /etc/secrets + +This will generate the new cookie hash/block keys, as well as other secrets Admin needs to run the Authorization server. + +* The **clientSecretFile** has been moved to **/etc/secrets/oidc_client_secret** so move that there. +* **claims** has been removed, just delete that. +* **authorizeUrl** and **tokenUrl** are no longer necessary. +* The **baseUrl** for the external Authorization Server is now in the **appAuth** section. +* The **thirdPartyConfig** has been moved to **appAuth** as well. +* **redirectUrl** has been defaulted to **/console**. If that's the value you want, then you no longer need this setting. + +From Propeller side, you might have a configuration section that looks like this: + +.. code-block:: yaml + + admin: + endpoint: dns:///mycompany.domain.com + useAuth: true + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret + tokenUrl: https://demo.nuclyde.io/oauth2/token + scopes: + - all + +This can now be simplified to: + +.. code-block:: yaml + + admin: + endpoint: dns:///mycompany.domain.com + # If you are using the built-in authorization server, you can delete the following two lines: + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret + +Specifically, + +* **useAuth** is deprecated and will be removed in a future version. Auth requirement will be discovered through an anonymous admin discovery call. +* **tokenUrl** and **scopes** will also be discovered through a metadata call. +* **clientId** and **clientSecretLocation** have defaults that work out of the box with the built-in authorization server (e.g. if you setup Google OpenID Connect). diff --git a/rsts/howto/authentication/setup.rst b/rsts/howto/authentication/setup.rst new file mode 100644 index 0000000000..e064637af6 --- /dev/null +++ b/rsts/howto/authentication/setup.rst @@ -0,0 +1,252 @@ +.. _howto_authentication_setup: + +############################ +How to Set Up Authentication +############################ + +***************** +IdP Configuration +***************** +Flyte Admin requires that the application in your identity provider be configured as a web client (i.e. with a client secret). We recommend allowing the application to be issued a refresh token to avoid interrupting the user's flow by frequently redirecting to the IdP. + +************************* +Flyte Admin Configuration +************************* +Please refer to the `inline documentation `_ on the ``Config`` object in the ``auth`` package for a discussion on the settings required. + +********************** +Example Configurations +********************** + +Below are listed some canonical examples of how to set up some of the common IdPs to secure your Fyte services. OpenID Connect enables users to authenticate, in the +browser, with an existing IdP. Flyte also allows connecting to an external OAuth2 Authorization Server to allow centrally managed third party app access. + +OpenID Connect +=============== + +OpenID Connect allows users to authenticate to Flyte in their browser using a familiar authentication provider (perhaps an organization-wide configured IdP). +Flyte supports connecting with external OIdC providers. Here are some examples for how to set these up: + +Google OpenID Connect +--------------------- + +Follow `Google Docs `__ on how to configure the IdP for OpenIDConnect. + +.. note:: + + Make sure to create an OAuth2 Client Credential. The `client_id` and `client_secret` will be needed in the following + steps. + +Okta OpenID Connect +------------------- + +Okta supports OpenID Connect protocol and the creation of custom OAuth2 Authorization Servers, allowing it to act as both the user and apps IdP. +It offers more detailed control on access policies, user consent, and app management. + +1. If you don't already have an Okta account, sign up for one `here `__. +2. Create an app (choose Web for the platform) and OpenID Connect for the sign-on method. +3. Add Login redirect URIs (e.g. http://localhost:30081/callback for sandbox or https:///callback) +4. OPTIONAL: Add logout redirect URIs (e.g. http://localhost:30081/logout for sandbox) +5. Write down the Client ID and Client Secret + +KeyCloak OpenID Connect +------------------------ + +`KeyCloak `__ is an open source solution for authentication, it supports both OpenID Connect and OAuth2 protocols (among others). +KeyCloak can be configured to be both the OpenID Connect and OAuth2 Authorization Server provider for Flyte. + +Apply configuration +------------------- + +1. Store the `client_secret` in a k8s secret as follows: + +.. prompt:: bash + + kubectl edit secret -n flyte flyte-admin-auth + +Add a new key under `stringData`: + +.. code-block:: yaml + + stringData: + oidc_client_secret: from the previous step + data: + ... + +Save and close your editor. + +2. Edit FlyteAdmin config to add `client_id` and configure auth as follows: + +.. prompt:: bash + + kubectl get deploy -n flyte flyteadmin -o yaml | grep "name: flyte-admin-config" + +This will output the name of the config map where the `client_id` needs to go. + +.. prompt:: bash + + kubectl edit configmap -n flyte + +Follow the inline comments to make the necessary changes: + +.. code-block:: yaml + + server: + ... + security: + secure: false + # 1. Enable Auth by turning useAuth to true + useAuth: true + ... + auth: + userAuth: + openId: + # 2. Put the URL of the OpenID Connect provider. + # baseUrl: https://accounts.google.com # Uncomment for Google + baseUrl: https://dev-14186422.okta.com/oauth2/default # Okta with a custom Authorization Server + scopes: + - profile + - openid + # - offline_access # Uncomment if OIdC supports issuing refresh tokens. + # 3. Replace with the client ID created for Flyte. + clientId: 0oakkheteNjCMERst5d6 + +Save and exit your editor. + +3. Restart `flyteadmin` for the changes to take effect: + +.. prompt:: bash + + kubectl rollout restart deployment/flyteadmin -n flyte + +OAuth2 Authorization Server +=========================== + +An OAuth2 Authorization Server allows external clients to request to authenticate and act on behalf of users (or as their own identities). Having +an OAuth2 Authorization Server enables Flyte administrators control over which apps can be installed and what scopes they are allowed to request or be granted (i.e. what privileges can they assume). + +Flyte comes with a built-in authorization server that can be statically configured with a set of clients to request and act on behalf of the user. +The default clients are defined `here `__ +and the corresponding section can be modified through configs. + +To set up an external OAuth2 Authorization Server, please follow the instructions below: + +Okta IdP +-------- + +1. Under security -> API, click `Add Authorization Server`. Set the audience to the public URL of flyte admin (e.g. https://flyte.mycompany.io/). +2. Under `Access Policies`, click `Add New Access Policy` and walk through the wizard to allow access to the authorization server. +3. Under `Scopes`, click `Add Scope`. Set the name to `all` (required) and check `Require user consent for this scope` (recommended). +4. Create 2 apps (for fltyectl and flytepropeller) to enable these clients to communicate with the service. + Flytectl should be created as a `native client`. + FlytePropeller should be created as an `OAuth Service` and note the client ID and client Secrets provided. + +KeyCloak IdP +------------ + +`KeyCloak `__ is an open source solution for authentication, it supports both OpenID Connect and OAuth2 protocols (among others). +KeyCloak can be configured to be both the OpenID Connect and OAuth2 Authorization Server provider for flyte. + +Apply Configurations +-------------------- + +1. It is possible to direct Flyte admin to use an external authorization server. To do so, edit the same config map once more and follow these changes: + +.. code-block:: yaml + + auth: + appAuth: + # 1. Choose External if you will use an external Authorization Server (e.g. a Custom Authorization server in Okta) + # Choose Self (or omit the value) to use Flyte Admin's internal (albeit limited) Authorization Server. + authServerType: External + + # 2. Optional: Set external auth server baseUrl if different from OpenId baseUrl. + externalAuthServer: + baseUrl: https://dev-14186422.okta.com/oauth2/auskngnn7uBViQq6b5d6 + thirdPartyConfig: + flyteClient: + # 3. Replace with a new Native Client ID provisioned in the custom authorization server + clientId: flytectl + + redirectUri: https://localhost:53593/callback + + # 4. "all" is a required scope and must be configured in the custom authorization server + scopes: + - offline + - all + userAuth: + openId: + baseUrl: https://dev-14186422.okta.com/oauth2/auskngnn7uBViQq6b5d6 # Okta with a custom Authorization Server + scopes: + - profile + - openid + # - offline_access # Uncomment if OIdC supports issuing refresh tokens. + clientId: 0oakkheteNjCMERst5d6 + +1. Store flyte propeller's `client_secret` in a k8s secret as follows: + +.. prompt:: bash + + kubectl edit secret -n flyte flyte-propeller-auth + +Add a new key under `stringData`: + +.. code-block:: yaml + + stringData: + client_secret: from the previous step + data: + ... + +Save and close your editor. + +2. Edit FlytePropeller config to add `client_id` and configure auth as follows: + +.. prompt:: bash + + kubectl get deploy -n flyte flytepropeller -o yaml | grep "name: flyte-propeller-config" + +This will output the name of the config map where the `client_id` needs to go. + +.. prompt:: bash + + kubectl edit configmap -n flyte + +Follow the inline comments to make the necessary changes: + +.. code-block:: yaml + + admin: + # 1. Replace with the client_id provided by the OAuth2 Authorization Server above. + clientId: flytepropeller + +Close the editor + +3. Restart `flytepropeller` for the changes to take effect: + +.. prompt:: bash + + kubectl rollout restart deployment/flytepropeller -n flyte + +*************************** +Continuous Integration - CI +*************************** + +If your organization does any automated registration, then you'll need to authenticate with the `basic authentication `_ flow (username and password effectively). After retrieving an access token from the IDP, you can send it along to Flyte Admin as usual. + +Flytekit configuration variables are automatically designed to look up values from relevant environment variables. However, to aid with continuous integration use-cases, Flytekit configuration can also reference other environment variables. + +For instance, if your CI system is not capable of setting custom environment variables like ``FLYTE_CREDENTIALS_CLIENT_SECRET`` but does set the necessary settings under a different variable, you may use ``export FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_ENV_VAR=OTHER_ENV_VARIABLE`` to redirect the lookup. A ``FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_FILE`` redirect is available as well, where the value should be the full path to the file containing the value for the configuration setting, in this case, the client secret. We found this redirect behavior necessary when setting up registration within our own CI pipelines. + +The following is a listing of the Flytekit configuration values we set in CI, along with a brief explanation. + +* ``FLYTE_CREDENTIALS_CLIENT_ID`` and ``FLYTE_CREDENTIALS_CLIENT_SECRET`` + When using basic authentication, this is the username and password. +* ``export FLYTE_CREDENTIALS_AUTH_MODE=basic`` + This tells the SDK to use basic authentication. If not set, Flytekit will assume you want to use the standard OAuth based three-legged flow. +* ``export FLYTE_CREDENTIALS_AUTHORIZATION_METADATA_KEY=text`` + At Lyft, the value is set to conform to this `header config `_ on the Admin side. +* ``export FLYTE_CREDENTIALS_SCOPE=text`` + When using basic authentication, you'll need to specify a scope to the IDP (instead of ``openid``, which is only for OAuth). Set that here. +* ``export FLYTE_PLATFORM_AUTH=True`` + Set this to force Flytekit to use authentication, even if not required by Admin. This is useful as you're rolling out the requirement. From 719ead2734adb4c586c9115026ddc656c5408b29 Mon Sep 17 00:00:00 2001 From: Flyte Bot Date: Wed, 5 May 2021 16:26:15 -0700 Subject: [PATCH 66/92] Flyte milestone v0.13 release (#979) Signed-off-by: Haytham Abuelfutuh --- CHANGELOG/CHANGELOG-v0.13.0.md | 18 ++++++++++++++++++ README.md | 5 ++--- deployment/eks/flyte_generated.yaml | 12 ++++++------ deployment/gcp/flyte_generated.yaml | 12 ++++++------ deployment/sandbox/flyte_generated.yaml | 12 ++++++------ deployment/test/flyte_generated.yaml | 10 +++++----- end2end/tests/endtoend.yaml | 2 +- kustomize/overlays/eks/kustomization.yaml | 2 +- kustomize/overlays/gcp/kustomization.yaml | 2 +- kustomize/overlays/sandbox/kustomization.yaml | 2 +- kustomize/overlays/test/kustomization.yaml | 2 +- 11 files changed, 48 insertions(+), 31 deletions(-) create mode 100644 CHANGELOG/CHANGELOG-v0.13.0.md diff --git a/CHANGELOG/CHANGELOG-v0.13.0.md b/CHANGELOG/CHANGELOG-v0.13.0.md new file mode 100644 index 0000000000..8e834c660a --- /dev/null +++ b/CHANGELOG/CHANGELOG-v0.13.0.md @@ -0,0 +1,18 @@ +# Flyte v0.13.0 + +## Platform +- Oauth2 support with SingleSignOn and configuration examples for popular IDP's now available in Flyte. + Please see the updated [information and description of the feature](https://github.com/flyteorg/flyte/blob/master/rsts/howto/authentication/index.rst), and the [setup information](https://github.com/flyteorg/flyte/blob/master/rsts/howto/authentication/setup.rst) + **Attention: If using Auth already - this is a BREAKING change**. refer to the [migration guide](https://github.com/flyteorg/flyte/blob/master/rsts/howto/authentication/migration.rst) to update configuration to ensure Admin continues to work. (No migration needed if auth is not turned on.) + +* Backend improvements to support dynamic workflow visualization (in future releases). +* Lot of features added to [flytectl](https://flytectl.readthedocs.io/en/latest/) . +* Documentation site overhaul and redesign (more in progress) + +## Flytekit +The first two features should be considered beta and subject to change +* First cut of the control plane classes to replace the old `Sdk...` classes. These classes provide programmatic access to a Flyte backend. +* New paradigm for flytekit-only plugins +* SqlAlchemy/Dolt plugins. + +Please see the [flytekit release](https://github.com/flyteorg/flytekit/releases/tag/v0.18.0) for the full list and more details. diff --git a/README.md b/README.md index 90dcaac6fe..11dbdea583 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Flyte is a production-grade, container-native, type-safe workfl Commit Activity Commits since Last Release GitHub Milestones Completed - GitHub Next Milestone Percentage + GitHub Next Milestone Percentage Docs @@ -227,8 +227,7 @@ Here are the resources that would help you get a better understanding of Flyte. ### Blog Posts -1. [Introducing Flyte: A Cloud Native Machine Learning and Data Processing Platform](https://eng.lyft.com/introducing-flyte-cloud-native-machine-learning-and-data-processing-platform-fb2bb3046a59) -2. [Building a Gateway to Flyte](https://eng.lyft.com/building-a-gateway-to-flyte-474b451b32c8) +[Blog site](https://blog.flyte.org/) ### Podcasts diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index 24dbf837d2..c07857dea5 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -8740,7 +8740,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8787,7 +8787,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8804,7 +8804,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8818,7 +8818,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8838,7 +8838,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: generate-secrets volumeMounts: @@ -9120,7 +9120,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml index 58f77c77f8..a4b30d8a68 100644 --- a/deployment/gcp/flyte_generated.yaml +++ b/deployment/gcp/flyte_generated.yaml @@ -8782,7 +8782,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8829,7 +8829,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8846,7 +8846,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8860,7 +8860,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8880,7 +8880,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: generate-secrets volumeMounts: @@ -9162,7 +9162,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index fda4d1b493..1391743266 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -2883,7 +2883,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -2936,7 +2936,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -2952,7 +2952,7 @@ spec: - seed-projects - flytesnacks - flyteexamples - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -2966,7 +2966,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -2986,7 +2986,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: generate-secrets volumeMounts: @@ -3395,7 +3395,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 536197903f..93e13fe761 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -737,7 +737,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -790,7 +790,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -806,7 +806,7 @@ spec: - seed-projects - flytetester - flytesnacks - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -820,7 +820,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -840,7 +840,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: ghcr.io/flyteorg/flyteadmin:v0.5.3 + image: ghcr.io/flyteorg/flyteadmin:v0.5.4 imagePullPolicy: IfNotPresent name: generate-secrets volumeMounts: diff --git a/end2end/tests/endtoend.yaml b/end2end/tests/endtoend.yaml index 7cd2cb6a50..cd199c4e3c 100644 --- a/end2end/tests/endtoend.yaml +++ b/end2end/tests/endtoend.yaml @@ -11,7 +11,7 @@ spec: command: - bash - -c - image: ghcr.io/flyteorg/flytetools:2638d4444add18096a688bc458d963cd3997e52e + image: ghcr.io/flyteorg/flytetools:a1ba0a032ad82d63177738129a8cebee2d1b13bc imagePullPolicy: IfNotPresent name: flytetester resources: diff --git a/kustomize/overlays/eks/kustomization.yaml b/kustomize/overlays/eks/kustomization.yaml index ed9f631986..adf72480a4 100644 --- a/kustomize/overlays/eks/kustomization.yaml +++ b/kustomize/overlays/eks/kustomization.yaml @@ -21,7 +21,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.5.3 # FLYTEADMIN_TAG override the tag + newTag: v0.5.4 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name diff --git a/kustomize/overlays/gcp/kustomization.yaml b/kustomize/overlays/gcp/kustomization.yaml index f1f7bccb5c..7cedcd55b2 100644 --- a/kustomize/overlays/gcp/kustomization.yaml +++ b/kustomize/overlays/gcp/kustomization.yaml @@ -23,7 +23,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.5.3 # FLYTEADMIN_TAG override the tag + newTag: v0.5.4 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name diff --git a/kustomize/overlays/sandbox/kustomization.yaml b/kustomize/overlays/sandbox/kustomization.yaml index 68eb101074..ca49cdd445 100644 --- a/kustomize/overlays/sandbox/kustomization.yaml +++ b/kustomize/overlays/sandbox/kustomization.yaml @@ -25,7 +25,7 @@ resources: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.5.3 # FLYTEADMIN_TAG override the tag + newTag: v0.5.4 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name diff --git a/kustomize/overlays/test/kustomization.yaml b/kustomize/overlays/test/kustomization.yaml index b9aab66cea..6a90ebc171 100644 --- a/kustomize/overlays/test/kustomization.yaml +++ b/kustomize/overlays/test/kustomization.yaml @@ -19,7 +19,7 @@ bases: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.5.3 # FLYTEADMIN_TAG override the tag + newTag: v0.5.4 # FLYTEADMIN_TAG override the tag newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name From 62f7ec464477e5099654f9f715a7b0ef6e5c26a5 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 5 May 2021 17:56:44 -0700 Subject: [PATCH 67/92] Add Auth Ingress endpoints (#982) Signed-off-by: Haytham Abuelfutuh --- deployment/eks/flyte_generated.yaml | 14 ++++++++++++++ deployment/gcp/flyte_generated.yaml | 14 ++++++++++++++ deployment/sandbox/flyte_generated.yaml | 14 ++++++++++++++ kustomize/base/ingress/ingress.yaml | 14 ++++++++++++++ .../overlays/eks/flyte/ingress/ingress_grpc.yaml | 14 ++++++++++++++ 5 files changed, 70 insertions(+) diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index c07857dea5..4b93b2a1b9 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -9379,3 +9379,17 @@ spec: number: 81 path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml index a4b30d8a68..ec65992910 100644 --- a/deployment/gcp/flyte_generated.yaml +++ b/deployment/gcp/flyte_generated.yaml @@ -9279,6 +9279,20 @@ spec: number: 81 path: /flyteidl.service.AdminService pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific - backend: service: name: flyteadmin diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 1391743266..ab9754f08e 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -3657,6 +3657,20 @@ spec: number: 81 path: /flyteidl.service.AdminService pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific - backend: service: name: flyteadmin diff --git a/kustomize/base/ingress/ingress.yaml b/kustomize/base/ingress/ingress.yaml index 041421259f..f447ef9a7d 100644 --- a/kustomize/base/ingress/ingress.yaml +++ b/kustomize/base/ingress/ingress.yaml @@ -26,6 +26,20 @@ spec: name: flyteadmin port: number: 81 + - path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 81 + - path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 81 # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi pathType: ImplementationSpecific diff --git a/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml b/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml index f7ea333fb8..ce9c2e8892 100644 --- a/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml +++ b/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml @@ -29,3 +29,17 @@ spec: number: 81 path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific From 14cbc436ea83c8c0ac82b6d3b3963b5d6f03466b Mon Sep 17 00:00:00 2001 From: Ketan Umare <16888709+kumare3@users.noreply.github.com> Date: Thu, 6 May 2021 20:31:28 -0700 Subject: [PATCH 68/92] Updated Readme.md with CII Badge (#990) Signed-off-by: Haytham Abuelfutuh --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 11dbdea583..c76b77e6c2 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Flyte is a production-grade, container-native, type-safe workfl Docs + Twitter Follow Slack Status From 463230010cf672862c337bb7fed9b200e0ee668e Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss <77798312+pmahindrakar-oss@users.noreply.github.com> Date: Fri, 7 May 2021 10:02:35 +0530 Subject: [PATCH 69/92] Wait for deployments to be created using rollout before checking for availability (#991) Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- docker/sandbox/wait-for-flyte.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/sandbox/wait-for-flyte.sh b/docker/sandbox/wait-for-flyte.sh index c99e722944..2a35b3cb6f 100755 --- a/docker/sandbox/wait-for-flyte.sh +++ b/docker/sandbox/wait-for-flyte.sh @@ -12,6 +12,12 @@ until k3s kubectl explain deployment &> /dev/null; do sleep 1; done # Wait for Flyte namespace to be created. This is necessary for the next step. timeout 600 sh -c "until k3s kubectl get namespace flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Flyte namespace to be created"; exit 1 ) +# Wait for Flyte deployment to be created. This is necessary for the next step. +timeout 600 sh -c "until k3s kubectl rollout status deployment datacatalog -n flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the datacatalog rollout to be created"; exit 1 ) +timeout 600 sh -c "until k3s kubectl rollout status deployment flyteadmin -n flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the flyteadmin rollout to be created"; exit 1 ) +timeout 600 sh -c "until k3s kubectl rollout status deployment flyteconsole -n flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the flyteconsole rollout to be created"; exit 1 ) +timeout 600 sh -c "until k3s kubectl rollout status deployment flytepropeller -n flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the flytepropeller rollout to be created"; exit 1 ) + # Wait for flyte deployment k3s kubectl wait --for=condition=available deployment/datacatalog deployment/flyteadmin deployment/flyteconsole deployment/flytepropeller -n flyte --timeout=10m || ( echo >&2 "Timed out while waiting for the Flyte deployment to start"; exit 1 ) From cad2857ca84cda2bfef008066b3c22dbb55e6775 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 7 May 2021 16:51:57 -0400 Subject: [PATCH 70/92] update links to autogenerated flytesnacks pages (#999) * update links to autogenerated flytesnacks pages due to https://github.com/flyteorg/flytesnacks/pull/200 we need to update a bunch of links referring to auto-generated flytesnacks examples Signed-off-by: cosmicBboy * update link Signed-off-by: cosmicBboy * update Signed-off-by: cosmicBboy * update links cookbook -> user guide Signed-off-by: cosmicBboy Signed-off-by: Haytham Abuelfutuh --- README.md | 34 ++++++++++++------------- rsts/howto/enable_and_use_schedules.rst | 2 +- rsts/howto/launchplans.rst | 2 +- rsts/howto/notifications.rst | 4 +-- rsts/index.rst | 2 +- rsts/plugins/spark_k8s.rst | 2 +- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index c76b77e6c2..031002a925 100644 --- a/README.md +++ b/README.md @@ -117,25 +117,25 @@ To dig deeper into Flyte, refer to the [Documentation](https://docs.flyte.org/en - Supports multiple **[data types](https://docs.flyte.org/projects/cookbook/en/latest/core.html)** for machine learning and data processing pipelines, such as Blobs (images, arbitrary files), Directories, Schema (columnar structured data), collections, maps etc. - Memoization and Lineage tracking - Workflow features: - - Start with one task, convert to a pipeline, attach **[multiple schedules](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_remote_flyte/lp_schedules.html)**, trigger using a programmatic API, or on-demand + - Start with one task, convert to a pipeline, attach **[multiple schedules](https://docs.flyte.org/projects/cookbook/en/latest/auto/deployment/workflow/lp_schedules.html)**, trigger using a programmatic API, or on-demand - Parallel step execution - - Extensible backend to add [customized plugin](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_advanced/custom_task_plugin.html) experience (with simplified user experience) - - **[Branching](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/run_conditions.html)** - - Inline **[subworkflows](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/subworkflows.html)** (a workflow can be embeded within one node of the top level workflow) + - Extensible backend to add **[customized plugin](https://docs.flyte.org/projects/cookbook/en/latest/auto/core/extend_flyte/custom_task_plugin.html)** experience (with simplified user experience) + - **[Branching](https://docs.flyte.org/projects/cookbook/en/latest/auto/core/control_flow/run_conditions.html)** + - Inline **[subworkflows](https://docs.flyte.org/projects/cookbook/en/latest/auto/core/control_flow/subworkflows.html)** (a workflow can be embeded within one node of the top level workflow) - Distributed **remote child workflows** (a remote workflow can be triggered and statically verified at compile time) - - **[Array Tasks](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/map_task.html)** (map a function over a large dataset -- ensures controlled execution of thousands of containers) - - **[Dynamic workflow](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/dynamics.html)** creation and execution with runtime type safety + - **[Array Tasks](https://docs.flyte.org/projects/cookbook/en/latest/auto/core/control_flow/map_task.html)** (map a function over a large dataset -- ensures controlled execution of thousands of containers) + - **[Dynamic workflow](https://docs.flyte.org/projects/cookbook/en/latest/auto/core/control_flow/dynamics.html)** creation and execution with runtime type safety - Container side [plugins](https://docs.flyte.org/projects/cookbook/en/latest/plugins.html) with first class support in Python - - _PreAlpha_: Arbitrary flytekit-less containers supported ([RawContainer](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/raw_container.html)) -- Guaranteed **[reproducibility](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_basic/task_cache.html)** of pipelines via: + - _PreAlpha_: Arbitrary flytekit-less containers supported ([RawContainer](https://docs.flyte.org/projects/cookbook/en/latest/auto/core/containerization/raw_container.html)) +- Guaranteed **[reproducibility](https://docs.flyte.org/projects/cookbook/en/latest/auto/core/flyte_basics/task_cache.html)** of pipelines via: - Versioned data, code and models - Automatically tracked executions - Declarative pipelines - **Multi cloud support** (AWS, GCP and others) - Extensible core, modularized, and deep observability - Automated notifications to Slack, Email, and Pagerduty -- [Multi K8s cluster support](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_pod/index.html) -- Out of the box support to run **[Spark jobs on K8s](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_k8s_spark/index.html)**, **[Hive queries](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_hive/index.html)**, etc. +- [Multi K8s cluster support](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/pod/index.html) +- Out of the box support to run **[Spark jobs on K8s](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/k8s_spark/index.html)**, **[Hive queries](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/index.html)**, etc. - Snappy Console - Python CLI and Golang CLI (flytectl) - Written in **Golang** and optimized for large running jobs' performance @@ -150,17 +150,17 @@ To dig deeper into Flyte, refer to the [Documentation](https://docs.flyte.org/en ## 🔌 Available Plugins - Containers -- [K8s Pods](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_pod/index.html) +- [K8s Pods](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/pod/index.html) - AWS Batch Arrays - K8s Pod Arrays -- K8s Spark (native [Pyspark](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_k8s_spark/index.html) and Java/Scala) +- K8s Spark (native [Pyspark](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/k8s_spark/index.html) and Java/Scala) - AWS Athena -- [Qubole Hive](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_hive/index.html) +- [Qubole Hive](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/index.html) - Presto Queries -- Distributed Pytorch (K8s Native) -- [Pytorch Operator](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_kfpytorch/index.html) -- Sagemaker([builtin algorithms](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_sagemaker_training/sagemaker_builtin_algo_training.html) & [custom models](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_sagemaker_training/sagemaker_custom_training.html)) -- Distributed Tensorflow (K8s Native) - TFOperator -- Papermill notebook execution ([Python](https://github.com/lyft/flytekit/tree/master/plugins/papermill) and Spark) +- Distributed Pytorch (K8s Native) -- [Pytorch Operator](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/kfpytorch/index.html) +- Sagemaker([builtin algorithms](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/aws/sagemaker_training/sagemaker_builtin_algo_training.html) & [custom models](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/aws/sagemaker_training/sagemaker_custom_training.html)) +- Distributed Tensorflow (K8s Native) - [TFOperator](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/kftensorflow/index.html) +- Papermill notebook execution ([Python](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/flytekit_plugins/papermilltasks/index.html) and Spark) - Type safe and data checking for Pandas dataframe using Pandera ### In Queue diff --git a/rsts/howto/enable_and_use_schedules.rst b/rsts/howto/enable_and_use_schedules.rst index 9bc719282f..9cf9dd91bd 100644 --- a/rsts/howto/enable_and_use_schedules.rst +++ b/rsts/howto/enable_and_use_schedules.rst @@ -98,7 +98,7 @@ assigned to the ``triggered_time`` input you could define the following launch p fixed_inputs={"an_input": 3}, ) -Please see a more complete example in the :std:ref:`cookbook `. +Please see a more complete example in the :std:ref:`User Guide `. Activating a schedule ===================== diff --git a/rsts/howto/launchplans.rst b/rsts/howto/launchplans.rst index 017e642325..511af6f8a1 100644 --- a/rsts/howto/launchplans.rst +++ b/rsts/howto/launchplans.rst @@ -13,7 +13,7 @@ When to use launchplans? - I want to share my workflow to another user but also make sure that some inputs can be overridden if needed. - I want to share my workflow with another user but make sure that some inputs are never changed. -For preliminary examples on using launch plans in code, check out the canonical :std:ref:`cookbook ` examples. +For preliminary examples on using launch plans in code, check out the canonical :std:ref:`User Guide ` examples. Partial Inputs for Launchplans ============================== diff --git a/rsts/howto/notifications.rst b/rsts/howto/notifications.rst index d38d1702b9..e85414da00 100644 --- a/rsts/howto/notifications.rst +++ b/rsts/howto/notifications.rst @@ -30,7 +30,7 @@ For example # This launch plan triggers email notifications when the workflow execution it triggered reaches the phase `SUCCEEDED`. my_notifiying_lp = LaunchPlan.create( "my_notifiying_lp", - my_workflow_defintiion, + my_workflow_definition, default_inputs={"a": 4}, notifications=[ Email( @@ -41,7 +41,7 @@ For example ) -See detailed usage examples in the :std:ref:`cookbook ` +See detailed usage examples in the :std:ref:`User Guide ` Notifications can be combined with schedules to automatically alert you when a scheduled job succeeds or fails. diff --git a/rsts/index.rst b/rsts/index.rst index b3b0f6af9e..fa8fbe137e 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -56,7 +56,7 @@ Meet Flyte .. raw:: html -

The workflow automation platform for complex, mission-critical data and ML processes at scale

+

The workflow automation platform for complex, mission-critical data and ML processes at scale

Flyte is an open-source, container-native, structured programming and distributed processing platform. It enables highly concurrent, scalable and maintainable workflows for machine learning and data processing. diff --git a/rsts/plugins/spark_k8s.rst b/rsts/plugins/spark_k8s.rst index 772da4e598..7acc84342d 100644 --- a/rsts/plugins/spark_k8s.rst +++ b/rsts/plugins/spark_k8s.rst @@ -48,7 +48,7 @@ You can optionally configure the Plugin as per the - `backend Config Structure < Spark in Flytekit ======================== -For a more complete example refer to :std:ref:`Cookbook Spark Plugin ` +For a more complete example refer to the :std:ref:`User Guide ` #. Ensure you have ``flytekit>=0.16.0`` #. Enable Spark in backend, following the previous section. From 1a53db9e35a77a72fcefad2c05456224ff1aaa15 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Tue, 11 May 2021 11:27:09 +0530 Subject: [PATCH 71/92] Include troubleshooting guide in the docs (#987) * troubleshooting guide in the docs Signed-off-by: Samhita Alla * modified list index Signed-off-by: Samhita Alla * added disk space condition Signed-off-by: Samhita Alla * modified list index Signed-off-by: Samhita Alla * moved troubleshooting guide to community Signed-off-by: Samhita Alla Signed-off-by: Haytham Abuelfutuh --- rsts/community/troubleshoot.rst | 41 +++++++++++++++++++++++++++++++++ rsts/getting_started.rst | 3 +++ rsts/index.rst | 1 + 3 files changed, 45 insertions(+) create mode 100644 rsts/community/troubleshoot.rst diff --git a/rsts/community/troubleshoot.rst b/rsts/community/troubleshoot.rst new file mode 100644 index 0000000000..3b2d43aba4 --- /dev/null +++ b/rsts/community/troubleshoot.rst @@ -0,0 +1,41 @@ +.. _troubleshoot: + +Troubleshooting Guide +--------------------- + +.. admonition:: Why have we crafted this guide? + + Let go of overthinking; peep into this page. + +We've been working diligently to help users sort out issues. + +Here are a couple of techniques we believe would help you jump out of the pandora box quickly! + +* If the issue is related to the ``make start`` command: + - ``make start`` usually gets completed within five minutes (could take longer if you aren't in the United States). + - If ``make start`` results in a timeout issue: + .. code-block:: bash + + Starting Flyte sandbox + Waiting for Flyte to become ready... + Error from server (NotFound): deployments.apps "datacatalog" not found + Error from server (NotFound): deployments.apps "flyteadmin" not found + Error from server (NotFound): deployments.apps "flyteconsole" not found + Error from server (NotFound): deployments.apps "flytepropeller" not found + Timed out while waiting for the Flyte deployment to start + + You can run ``make teardown`` followed by the ``make start`` command. + - If the ``make start`` command isn't proceeding by any chance, check the pods' statuses -- run the command ``docker exec flyte-sandbox kubectl get po -A``. + - If you think a pod's crashing by any chance, describe the pod by running the command ``docker exec flyte-sandbox kubectl describe po -n flyte``. This gives a detailed overview of the pod's status. + - If Kubernetes reports a disk pressure issue: + + - Check the memory stats of the docker container using the command ``docker exec flyte-sandbox df -h``. + - Prune the images and volumes. + - Given there's less than 10% free disk space, Kubernetes, by default, throws the disk pressure error. + +.. NOTE:: + + More coming soon. Stay tuned 👀 + + +If the issue is still bugging you, contact us on `Slack `__. \ No newline at end of file diff --git a/rsts/getting_started.rst b/rsts/getting_started.rst index f21ddc9d7c..d027fa228a 100644 --- a/rsts/getting_started.rst +++ b/rsts/getting_started.rst @@ -80,6 +80,9 @@ Steps REGISTRY=ghcr.io/flyteorg make fast_register +.. note:: + If the images are to be re-built, run ``make register`` command. + 9. Visit `the console `__, click launch, and enter your name as the input. 10. Give it a minute and once it's done, check out "Inputs/Outputs" on the top right corner to see your updated greeting. diff --git a/rsts/index.rst b/rsts/index.rst index fa8fbe137e..5806d86e1e 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -33,6 +33,7 @@ Join the Community community/contribute community/roadmap + community/troubleshoot .. toctree:: :caption: API Reference From dc44f8d2128cbcfc69c96c7570843bb264a02b83 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 11 May 2021 09:31:31 -0700 Subject: [PATCH 72/92] Add data council video Signed-off-by: Haytham Abuelfutuh --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 031002a925..d6333c0761 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,7 @@ Here are the resources that would help you get a better understanding of Flyte. - re:invent 2019 - Implementing ML workflows with Kubernetes and Amazon Sagemaker [video](https://youtu.be/G-wzIQQJKaE) - Cloud-native machine learning at Lyft with AWS Batch and Amazon EKS [video](https://youtu.be/n_rRb8u1GSM) - OSS + ELC NA 2020 [splash](https://ossna2020.sched.com/event/313cec91aa38a430a25f9571039874b8) -- Datacouncil [splash](https://docs.google.com/document/d/1ZsCDOZ5ZJBPWzCNc45FhNtYQOxYHz0PAu9lrtDVnUpw/edit) +- Datacouncil [video](https://www.youtube.com/watch?v=1BjXC5TZAiI) | [splash](https://docs.google.com/document/d/1ZsCDOZ5ZJBPWzCNc45FhNtYQOxYHz0PAu9lrtDVnUpw/edit) - FB AI@Scale [Making MLOps & DataOps a reality](https://www.facebook.com/atscaleevents/videos/ai-scale-flyte-making-mlops-and-dataops-a-reality/1047312585732459/) - [GAIC 2020](http://www.globalbigdataconference.com/seattle/global-artificial-intelligence-virtual-conference-122/speaker-details/ketan-umare-113746.html) From e846c27007e80313033776eb8488e4856a33a206 Mon Sep 17 00:00:00 2001 From: pmahindrakar-oss <77798312+pmahindrakar-oss@users.noreply.github.com> Date: Wed, 12 May 2021 19:25:42 +0530 Subject: [PATCH 73/92] Added tip for make start to check troubleshooting page (#1023) * Added tip for make start to check troubleshooting page Signed-off-by: Prafulla Mahindrakar Signed-off-by: Haytham Abuelfutuh --- rsts/community/troubleshoot.rst | 18 ++++++++++++++---- rsts/getting_started.rst | 3 +++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/rsts/community/troubleshoot.rst b/rsts/community/troubleshoot.rst index 3b2d43aba4..9d9b821096 100644 --- a/rsts/community/troubleshoot.rst +++ b/rsts/community/troubleshoot.rst @@ -25,9 +25,19 @@ Here are a couple of techniques we believe would help you jump out of the pandor Timed out while waiting for the Flyte deployment to start You can run ``make teardown`` followed by the ``make start`` command. - - If the ``make start`` command isn't proceeding by any chance, check the pods' statuses -- run the command ``docker exec flyte-sandbox kubectl get po -A``. - - If you think a pod's crashing by any chance, describe the pod by running the command ``docker exec flyte-sandbox kubectl describe po -n flyte``. This gives a detailed overview of the pod's status. - - If Kubernetes reports a disk pressure issue: + + - If the ``make start`` command isn't proceeding by any chance, check the pods' statuses by run this command + + :: + + docker exec flyte-sandbox kubectl get po -A + - If you think a pod's crashing or getting evicted by any chance, describe the pod by running the command which gives detailed overview of pod's status + + :: + + docker exec flyte-sandbox kubectl describe po -n flyte + + - If Kubernetes reports a disk pressure issue: (node.kubernetes.io/disk-pressure) - Check the memory stats of the docker container using the command ``docker exec flyte-sandbox df -h``. - Prune the images and volumes. @@ -38,4 +48,4 @@ Here are a couple of techniques we believe would help you jump out of the pandor More coming soon. Stay tuned 👀 -If the issue is still bugging you, contact us on `Slack `__. \ No newline at end of file +If the issue is still bugging you, contact us on `Slack `__. diff --git a/rsts/getting_started.rst b/rsts/getting_started.rst index d027fa228a..ad750baf12 100644 --- a/rsts/getting_started.rst +++ b/rsts/getting_started.rst @@ -28,6 +28,9 @@ Steps make start +.. tip:: + In case make start throws any error please refer to the troubleshooting guide here `Troubleshoot `__ + 3. Take a minute to explore Flyte Console through the provided URL. .. image:: https://github.com/flyteorg/flyte/raw/static-resources/img/first-run-console-2.gif From 7d3e9764bbb119f2dac7ce45792abe17ff05f072 Mon Sep 17 00:00:00 2001 From: Ketan Umare <16888709+kumare3@users.noreply.github.com> Date: Wed, 12 May 2021 11:37:24 -0700 Subject: [PATCH 74/92] Update README.md (#1022) * Update README.md Signed-off-by: Ketan Umare * Update README (#1024) * updated readme Signed-off-by: Samhita Alla * updated readme Signed-off-by: Samhita Alla * removed css Signed-off-by: Samhita Alla * moved from html to md Signed-off-by: Samhita Alla * Update README.md Co-authored-by: Samhita Alla Co-authored-by: SandraGH5 <80421934+SandraGH5@users.noreply.github.com> Signed-off-by: Haytham Abuelfutuh --- README.md | 69 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index d6333c0761..46ba72be7b 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@

3n7q1E7jshN918}Dg_kKa( zyR0S|SPMG0$a|3Yq6p;LaDRLxy!MM9-_MXH$K2Ay&BXpsx>OW`sE<7^M1S(p^ALEM z0Gy_4;l4$Z$=a6tj3QQsc?T!Il3H8Wl2Kmqi%Iu6Wvr@oiQBKvN!BS2Sr~D4B z#$EIT@piX=AL34VJdorIkvofZ%$t4xn|Z&PWc{?aUIxKded>Du>DLiYQf|9v?Bl3N5g5rE}RS=+XVjNAhV`pgGHn#c zJ>_5dC&-N)R-~K^d40y!N4wsg)gUDQ%qM?P;6(TlB7)lPR59S=f$%%`uGN?MBy0az zemmf?dL z&Gy)J*O!&=z%^!;32&1cfPt?U+(>6FzUq;=Dd-QYpY7-K zVlOP58hnhNa`r@cuassJz*-CE#$ds&RY(30Y*wtlRVbh2JCPngi>kymKX0f$bf4oq z5I+4{`koc_n*?myL3G;3<-+O}BfedwK&?M`2pyh_RLhzAS4_jjZlukp^zQnR>`?q_P~px!11xbU>T* zTc3Zsh1@8MY6le?#Q4od5xrFfF$Ku$S06?WG06V!_*}qx64LQLn7CYkDVmBB(<#fg zSewDV?VVo4#~JJVBoJchtGVzac?TC-&dE+eO3=0AM#Fi<_V>KF95iL1@IAVb@s8^ z>dR(t&A~qkqmv!_Q7W@=Vv~CuxeTlw&$foNCy4k*HZkT0=YD^r!O$C$jOnPMi>kU7 zY4AGdRhzz%v0MzZgBATHE;Yt}R9ON!c$0u>-w;`IFu^-Y-?slt=`zu5+#U8XDTO24 z3wE!UOR;z!lXnCa(lGvA$ZqoIp8TJV;T1`v3$4!W7kd#mp?#^=>SD3~mR19#IC9C0 zM=Ejf+ZbEV*9mxIWEP8P5rjFZPm`VH`@iHGpd>1xdPDfahGbo8ELUpZjs&6f%Y!qe zs09R0PU>Om+ZbI*V6mN;7W6qGgdFZ~+qvx3uJ_xZ_R$vuB4+W!(Phk&9T7iH{{0c&wsS+Mu^{pga< zA}>j;J_MY!*Sy`;cN&Z=g1IO18nj;enHRMsV+waqd5Rg^Dc>}cOw*3-andtBc6+%5Lw!JPbL5g%3%{ix!M8#BS8qwz_R!O z?q@}Ay+|wzhvZ6b;C1UdT?CrWeQr5jUP;!@s}zg&Yp>fwm_)n}l&gxWE%*c~dhaa^ ztD2p(YObU3M@XRjl~QD2c})nnTHNTx8!kLJiR&@FZFFZr?B2JTTqeys@R*xi*++Xv znQTYQzh?!u|2!`r*|XgcF^zcJTZbVJ?YsFdSP<69?y87d!aQQ;YWR7dXs@rI?b1b& zL;-HUS;Itoqx0>PLkOBJ^h%-#WO6Bo%x;S>o*^PzWE!PZeM%70TXy6@XC~_PF^N^b z8)vkijuU@0pr8rL$G1xHy};i0bW2lEtgVo6yts5IW8*uQ{IdV8>a1*DX8Z)PS%Sw} zYcI*MDJUP{FvzOQnYA#UE%;DA7TfqIU+Ro>$1}{!x_6u%&-6ap=jBhr7g{d!VkIe3 zUt^YL&PQzU2Ex7wQGwxnL}j5j30~{GR@KDXYx`=}y%4(*@T)aS#+&Xjvqz;{^u$Hl zYSHLN0V&T9)=QpNmQc$-4NG00=QA^X#7oVpYltzHsjWEi<#9~w0X=bS0H*mXh-Szl z;_bkU?{W^JJLi+F`o!ms6`t=)8zUd=LmVXBB$%hJhLa8&8S1TQDk=f$J-^A`F(NX! zZTa%w0&uIY_a&0-LuuPq7Q$c0NhN>q$}AWro%-d2eqcO)QeMS;J*GhyFfn#K z-hWT+48FL~NTdNZ<6DF*u(Rh+5Ss&w{tSf-CO=2P_=+udw8prxotP0FSiA|?ELo%^v6O+umJzu z$piADl&cTk`RlMgfx*K$rwTZmGpxIQ>B=ugFFS-$Z_QTf?6;CDoS9+C9eMF*c|x#{xO#AOF$vueQ4Epojr5Vqc#Bq!SO*r~9E06kX$T>T^)2#A~1)s=HlU5hS5YhITDt~0ATZB>U*kE0CO=itoA*B({Z-E6OrR4 zjn9aBCSPn2LB;;!W@9H=m#ZhW#3515C+X@Ay+5ryK&Ig6B$&blSA{GRBlXw@;Oa$E z3r__?jSSH|-fi4z>WB;he05qJ>CLC9#JjC&FH0bsja;Kt1))^EchGDdED;i|Ilqu@ z0bMZ^3z4hb{opMj;c&zmO?#kOwQEpz&pCq1Nud8LJF3_@FgfN?y}{mo1NHTPnL(6nT@n33lat){nAJXR+-kMO zUC4Wqckh%8vHtIs+XxXqK=RxcP`#f-E?9rPjP6@*oH2Mhj?It{y+1IlgqjSl+6sGW z7=25Otjr@qHrX5w zNi?j?GETCRRko~SSIP_-Imcdcj+JBo@6YM^{{GMNeZF2V@p^jXbARsZzOMK6zTVe$ zcU~N!W-H-*q02A%V6v=!70bM_fZFspU?>;<7Jq9%)GFSoQQ}!T&o}2NCV_$at$o9q zamg*U7W;;VUFUP4SWO8f@8yj0pmnTNO$+$c8X{zN-#QCabT zweJ^HHy(B<*p8_hX?ce_j&fS$9IJ^n;m_f zeZeuTRXw2pE9KhH?Ym0^mj-vG0d7G0lS~>Uz`9F(@LAtzCaKGXUEue-)|ZE+bTr$6 z%9sS+qhhfl-VP(0i>AUnYv#tOrHkE6@m9t(y&;ze+}JC@?P6P19E%EX*n&bViS^;P znYMxsX0(A0m5nRZrP1!j!yTQXhPCEUKH99khQr|x6~i+~Bh(qBQQJ)00P$5ffl!r2 zg|G7grC90!k3jy~=4sQ28r^*fW9Z-K=Cqq76<`!k7BX^#S?E1x1 zeIJ9ZW4b2{wW_K8^40>J`s~Zi!oCl%)58MDPy6=MdvEwIsw_K-?~kX3oKr1XKptOg zx3nT39r_mU?DTl+cJO%l81Lu+^~+>nlRb}9RS@q7uU;I{JJ6+>i+8S8cP?xVs#edF z6CM)2T5k5psL064-tW4E%R`?Px9c@S>S_i|?N${aSKzAsG5%tl}dhV?60!y;`i^5s@PcRHs-!4ueL%ndY>sSWU}e3&I4?pY0Xha z-17!1>yT}<%~GM&`}{Flq31i6&Q4Of$z#0r?@n5?ZHCPxtnOPU@*Xvi8uAuQ(te)L zm+5|K>-PAUxq|4C`M7&GKeu<=f+Fq8vM|rkmjD%u)p%#O$IrEir+fK{h@AGIT1-kv z1#P(t2!IaU-i6Ex#bwp4-LOyETs7{uISr^<`@vkz8?Qh4L+SYNSgD84j(Y%%ZTwNMU#H<@c#vwha(?|z#coXp(b zmVJvnnfOaYBU!bvE}9e!n4Pxuo+28Dk~{ zyi5p~=pfJ8bwlU=D!x z+z*AB?1kVn53S=GrMS2_uP_|pdTYj~PR0+fn<~M=1;}gAAczFHVgdrTGN1Ihd|Gku zik2P6r#2tbYy}})>m@eNVQ+>tscW^-9c?ZMr_@n(ewds>9uk{mJ`ekJc*cfnX7lc3 z%PCNBKXJ#`T@ij|$DgRr*JJ88KV2k(&IHLtT(P4mS%ZEUamMB-dmVaooc&n0wMyBh z<>lW60Fn!)dB-;*=D9iOsA%;cMJr;F^&%ZiPmh|8l_bS>PF=5g9y$9$mp-KMY`Jhc z?+~A_w!qRHt#gZp?#x-EFK?OM40LLmCSo#kSu)=x8R?v9XJH4_iuH9F@rQHh26BnG z0||l3?VUqDf6FG>)sgsIoyU1yzgq`p>X}^DN9uWwT9pHWi(mE5^=VpnqLns~@#aZ% z&*FZq*T$|I=)__l&!N{IJ`!QSyWOF9Cq$X>_Pf@V|LMRAvt?Us|I&uB$B&T09pPjocmjPQm;eMjRrt$Ws2$XlEvL+WRLxZeqJ zkhuA|+kIy)$ou;^eXy5=sN9#y(C)a^$9V$&txroNh_3D^&(Bi5%sCD2sq7B6NobX^ z2y$h)mN`m=>M^HHk)jv@ONC($wEZQ}c5Cwcdr3Gdvc|DAVOu4^rI1(g&Mrt{Mp$+ep1v z41ETdCEr8`ROGWldF#g~(=)mz6|1Ll#w_yYPrf@XioT!hIc7K?vdFDiKoQQ_5V6DKVt< z3n9l3UCHxz^S)L_`TUnl92Q!gdPo!9?KScMjCH{4bVfx_i2$tfp*Rt|IV6>C&9}jQ z|C)TMdNWL3op^!@6w|N66xUAmUqX{weF*-Xy7g<;|H zl=u!Lb_ld0W<2=4OCDKLESoRgY*q*(-wti~JwuLSWdzY>dCv2!rRpDb!x)QBY+G$# z6MF8viy%4AdID6Ltwvd^72sF2Xk7m3 z6=Ku>D(ddyfl8Z}6xZVgm0ht0#jWP4s=F#t#N3Aziu4x_*S^20_Czm$UWxB78>V-- zYF5%=TM;8#JCE~p5cJC$G=Z}r_|Eua3aoDM=}CM>a;#bq{p{yu7o2S`Yw99PI&- zu+JS@xI3tVOm0GnBoLFtrqaS$C%cKzRkWQcHpx>dCxjG;Jd>4hA3aBo^J$WPQwJQ@*^@8dib%KOC)08qdwbCCb7#Wij7jGGXt z*cs{A$6m5qK6!?jyPkFV8%t8H_3TBf`zT*SVb6|q=I8rYmpkc_)S~84bZ?%`#K6qZm8B=1nrh!!bV+Zs}f7Lg}4SO62O=4j&YML$f_NwnEs^0x_?PQ(S^5vq57 z{L~BJ7G(*c5^U4ID%=^PriBMZo3gOdfHZPdS`mL0<4j z0m#i1DWi969@X5O-X()MI*6vt4NL^xWWl(Ld|)Z&wDqKs;9GJaGCZIGB*ZE$Zpa6- z3FT*PtI#`56tmn~^DX>N*+c;aqCER}ReEhmIJ3X#x^;FAGsYuJ zU>CQeX>p6boz{y%-I{ze&jJXt`%6ecDFJ}kA|Ui2K6q~6@D<%!c8v;G7 zL8w%)LG*C2@l29+l^V1!ll8l^5zW5>VQ+hPz_zGm-xBQkWOMy-5m0Zrq7bNfGP+ir zlYZ;hpJlBFcIq~t`wiaXJ~5`%Be=eftFX+kAt^R@dLwoz99!Px%-F1dkg)lkp!yy^ z>mn&X&l-L6NUA|*J+-d}yYn;4BJ>-E)bb&olVRFFmj0G8`$K$-mAL%;oux_oW4h*} zHi_+h(Z}AqV)RHd923Q8(5*#06;x?si3UUdumz;BLMk*MPDoJxCdJJD5+a(1HAA#& ztnyc${vgqEDe?D&RWj6weFqU(l?d-`A6?d!^U{96C5r8ef+NYv+K@sf?@sm%E9h31 z&k>NNyQ;dcd60JAK3m|)0vyRCdgO3%qR(8%U{yau!L*mk?*3gr5z-t3VARX$P z8+PpqHbXKM+L)x+)F|#ye9gz7`Fm99rO~N1j88==n!vA}fwXAe&?qOmcv56RKd>Kf zX?evkZm2|BRjXqDY~Li+#%d|EkHJ9e9wxY;iIuLv+;au4B@5yTh=1GUnBJk8`8smI zh;==ycbTwOFk<+{_ml*vnW}&TDgulyRM}!@3_Fe_lhQPk+%=RcMcTHFZ$JEAH)*We z)Ey)}!HB!1sdNE;`*An=z3nediaQ(u$S_F3H1wx!;Cm#uc39c#{`FHBz>op@ztB`# z@0YQF-lCaU&o8&K<8N`?M{fV$6uAc3ri(&cO|fCQkk{Z(+FjB|>aiJBd&7L8bB?;x-WAYSKZDLh{FQ`F}~gO%;Qh{+!%pSau==4&6rYhl{X9ESEBj17L{dYk3QXb$Wv1tfdx zqlN2p;+J4|Bv-u)2eiy2ooZP(9a8RwL|*pDF7lgEILAlnw-o1JN)@1pCGVsWYD@Oe zkyP0Wk!kSICvcg71OntIv2p<^as{;z0L2B6lWPI>J``|<3wWzbFpZWBaU){_I}Jj# zhz3mmEhJXxKTC&vkIz67*E)E1V|m9y&KKUE_g;TK%lqu5}BShh9Eyn1uh7N0uFGzO!$@p4A;WZXGdd zN0X-TjVe+vm;??2RS8j(NW1?2#6#Ac;xt_Lsb2SH@e)TW1>PY8t)?A zvl0=Z7Si-5-2Ju512?<_H|OZY87&ienk1m2u@@L3NSbSFB?fsiZ15d;8O2>OYEbJR1f=yVxR^XiO7VV~sZ;A21h9wdCTyzCLK^Zrio_u}o2LeK@uPUpnZj_Sw^RVr+T?}(VBr+?IvZ8}%FBJ?zx<@e9lTzwJG z4vW0nWlk;AP?u=X$=-yf%w*XGRYd|b2-XLP65+72*_NHxQ0LfBo=|<$XjnvI&Ze)?Fbc5m~$vWvARG(=f!h_jZXlJpt zxZ$Z-4UIbwM19(<=SCWg#~Mk-$&oA#p1Pc) z2CYd>sR_=JRoU!R{1CuQ2ksmpJ7xN=^Whq}qpb)Vhd4TSPtNlbpOf zl4y&lYdFqdV#FCt--*LK6x!N~N-?{Bh zxB8j z(Pa^U1raiAhXIl7JOEpH6y4?QIc?{ABU>45_ZfqY z^0*Kpe2WR7!2U>|xxW!Xb4swng}+ONr`P8A11`5c`WBPz_3cyN=C6Fp2}P}fMW~V6 zS7E-E!I@UH{w>7sp!BGUWP%Zj^FY^vZ=ANSzJ!!vX1}76hOQLLFLi?>bB{ z8ILxx*F|VBTKwK$4I4nfEz3U_k9YZ;8MlxFDBCRto}B_fHV=5Ts3n}SUwqXPMt3rYIAMCqfW0^-`Ok+9gBnNafffbTVG znnm$OKdKsufti%uNn*(9S+F#gyf78yqg<2^Ns&}oV;>CGEW0D5sIu1E>tfayk&b!tmW!U%A#w{x z0aWf;Ih(YTB0@eBJWUKEu_JGOTR6Qf`C%xGmGsqE*h5y81EJ!b48vs~5 zz!U^N+!0Tf9{`FY2TR$}$0H(~Z0AUJqZe^=3bAqQLw<_YN8}f%OWMgKM$bd=2vnJB zaapniq}&~Gte8W3EXM@JKLb)MBpWaSNFNKzq_SbhcXV#xn)EVo_i6Ot#Yf}+#t0d` z5+8lsb4aHF;05AQW`KR6%f~=rSeXc1()NSgBD=y-Lub{h5U8xQ-{Y^xoVilX{qvfl z)vYOpX<$cWv;s$+OF^O{e$-WNbn(A_IOzJHnD{X9)J4FDKCJ9JG)<7ZkQ!&D8hQ|Q z;lRPlimG<4(>!zdgKW(~UHoIe* z%wNAxG3(Q#BS>N7c(R1sETNM|N?dy4ZJic=0+(qVX5~mUv{t;_9ube|Uz+;(8BPHN zCjfbxg6=YYh}^&vp&tq%%?&m`kkyD6COw`w-R_vb?N@mD^3*VsZ|!pCgNz+DA@SfG zYSV^7T7nmxg_<>hLj`YLlYLRfWW$8ap+1@?0tMO<#DKW{#Ae21}>{@U+ojx?G~xor{5wm=ZPbT=aWS*1Os; zRh=nF={2g{(Z%k$IV}(Q&xK|i;8+3{T1?3x&~*shB^wxS0B}qhC1G(Gh=~M?8xO#Z zlYl0XzXR`E@L2PJGEIoZaBK132nBZFku{kj$eFI2~?OwMhra}>K$jXT~G@^!7=tHRe zljIHlWS~*+ADIT8ql{5LgBC81>RmU?ZX?{pev6VbX5p`63?xGU2EHb0qH0H1Sw<QpzQyY))U6}nuyGB{k z?i^~zFTSJ~ViQi$Lum#;N8U{G&X6xbN2qnNhw0NJ|DHBJ?FG2J7heeKir`g@-y-Px zK;s$I=K-}|0_#;sUG`i64&-g*1F`>T0qzvHh4hmbOHgfi-s#$8>#^)=C^VnAgnrG1 zQQ$eszYhF=e%H?;Q9x)5z3mnwT`GbN+4ThET>j+8hlqCrjksh4VupBHK<=oN)w!c6 zDi6$~$((2Oo3-oq$!5rzENkyL7VWYp{Xwq+M5i?EI}{kuviWBvi3C3ab`gvRID>K& z^f@{G4{W)vz{!Rw@&a!Q$hhdp2`9Tmvr8bX*9%ia3>0HkHwTtEZq(S!WMZmrb)(8V zC&EB~jF>n)nFzMl-kVw6RrbPFkkYmfj^BtR?w%V5u5h-+pVJn2AK+WT{SDw6Ho#@x z_h}%(i}R=)ouhKl7i{}j{+spk#LII}2ZG;B=yVy3Y?iZ7+iy; z+^G2~(z2-_%zu{a8c%4&$5T6rCvII>P&0{)P)8Z}I zWOVwxnPhr@B({VC$VS{yijepcP!JD6LtE&yWZ?_oJbs5zt?3u5wuoC<^2YbZ3SRdW zoFVVo*J2e%RB)ueCBi=i2;{_nTpvqlG%>^@gFYp0;-gwE=l@=gSko0;9VcEipzH*W z^m4li1JtmwergihT8t9tCjsyj`a6H|p{ZE@+in)GHGZCQy5;&FQ)I)ke$SfBJ>y-Q z*yi(_0ljSZL`V+~%*ZFM81B`tjkBwone6VaO)z@GEMK0$72r?(K?PXSh6N#?0g{qF zCoZm)sb3^H3~k9aXU z4ED|&5QTkeUFx{pdQ@w|`QLABS{=S)_7-qr;%yQLL#lLx=7OzS-=j}XOXN7>n53D zPC4(q?jB#VlnEonf#(PG57hB02x)e}Nf$o}P1vBubt&EaFOLv+>XBL$q`(bDG0KY* zAtC4m*@S2qwt^HIC_;&Xi4OJf%TIi~`fUGA%pKD)p7l;5k_W8Q7UDnKbrZ-Z+WQs- zg?uuk2g3plw`;xY_yw&ddsM5VN_B`}pgEct(*M-{CAp{v*&cC|!&@flw&Jr2KOT;o z8r?AGwB7CbG;Ztv%U2@|3)F8QMyf%_Mk{BA4ocTS-`*eOn2p3gn5@e!a+V533G!I9= z`0Q~uQ}0^sR5iBVJA);<$sNUCmZH-^^jAoDp(y|^^uCb;-T=r*K-n5BX(XJR`d?nT z`BbG^ltOz7(0jB58>_n>`eM8l@yfm!twL_w6>==o;~&6th(?#4z*@O<*($le%xN>+ zog>aq8=wnL42v>!O_SA)0*Ak#FM;@|^CkI&q;cE`?MR(c0s7f|tS10N1ZB*GvB5EDu8B~y*_XO?=sBE-@ToP<)6FZ1!e>5EYXRs)8%oZVFA z7Z0kVP7T_CT3|WolnGNHKzYa%IOaz?fD-`Xa^sfZ9*7o*8V^V7bP&Fif=GxiCCg`a z|Lq*4Cw(v!q?mnBrUZ@+qVgf~SK=>R4PxQg)dP*7iKgpu3d}2BhPljTzshJOzb4u2 z7RDsk-hE0d+k2PvfP@^?i*!Q8NS%*{W!IlU*}n*#eCyiD*{yvs*z%dCW64w1>TDw5 zz*WH1W`Lc~U=X(8_~ap26h1vCLZvCI<+ZVFMCD` zf`=q0Ag-d0Kq1{AR04o>E)q0D!f4vL0i*Me_^PrFoC**d5j6oE3Q)l-acI*C>Qhid zfKH?kIFSdO)M3pb30$BAF;fkeC=ZM4tQg@x%SuEfqJHpVX8UFcEgi-q!yWZd?t*Ir z6A#<3#aRz`dYq>@e<|wBik!(rinWApJ+8$uyzzs)nv<;+SZNY#V zhUUy18x$IGr%o^XmjrC%#A@tbD_m1<($>2?{lc`{vye{R6{Js-Ao+y02eej!<*-wS zfjkmaZh$*hAU^njxDfphmZqZ%Krv-NI}?f(fJy}s)-r~c0@^n}z{KbF1JM=PZ8unM z*EesUTF2y5D___+?+xg`72=bTK3N-_%(|Js1n{wJkLUNB=StJ>p*HKCNv2ynAdH@_b0^zQMCGMjZeVoNh;bip64A2m`uISh6LI^= zGiw9vU$|RA5@q@w+SXSf=Lc6Bf`qW$l<42U)MKgeW17{_cdqvWEPQgg zs2s{6TUQy34wQOc>r~BJW-3#s6gPG;$@?)U(?j;)%)d z+}g>w53^n959Q?S9?zm}N62;`sw?bMS)Q)|$ zn@gC=Jnw(!N?%8}JJfD5$`k55)CBT>1H)}A=p)Yu>A6AbH3`b%P_t}dwOF5BizsK- zk6UXTw^kB|JJ-`Ff4WUDLY_^D^q_%NbK**O^ojs0&eK^=<|i{e)Qz>&gq7WEP*G(5 zrewJiL|1~!!!05ev6F;UoDkC|=$96j{fqJ6V+y z4d`k#W6LBWh8RK6w1{chc?*@50Yn2G-V`ABf(ES2aJ^&sAAE$L`scjC70Pv2_YaEz znS*{_VrU`)D5|!cwH@yScN;+)1{yR0W(0k{ZouvNHgVN_9R%!P5LAHL951@X2ndD! zAG&THj9%R&iCWP}S@mU9s*!w9Lu~;wL&ypyh1)yl9czuac*ifVET~0@?kvhQyMK!0 zAOZX)K+aQ90Jz*mlQ4l_=T5QRx+-EliRh*Y@{j!VPH<)D!gDSo-as4$o&p!bWm^Qw z6BsCchj6L%V zyK|{O;OGKhwV+o+LD4|mxUb&?H!Rkr6Hksi+y=F>Bf(=xuMI@rCCx-CEoV^v#Jf-@ zi=U^^D+%e^Riqwg6zOq~FY#b=ap$f?(p0%Do{f_Wf>9|1+7aZqfOaIq-;C)=JOJ0S zlfoZEv)6iu7CnhW#bN_L5wCNfiUhq-s(RXh!E8qfvAHRXyZW`25_O~Ya-E6N?_20qD7p=0&Cxc8_TgfWW}cJ z?2`U3Xg3`{5Qh>L@6=w!5Ty882x6*y%lJv*4C?{z(hhM_wxvL!49Y})phpB0**Iw- z(^Lk}OoRkY!weXOkGPq~Nz==o;OLj4R+3{6$IMilX0D7E)x~aR)dCCN<6XNbyRnB50J;IRBCj>fB$$#YhWe4Aj9ZT2cdB& z(GzdqISsYu0qh495sFYU3wZl@Ap3-eNsOdGssMXb$!5$Q`-XELZhG(*7uVFecey?L zmL_Y(=DG5&FM$`JfHS46MO}uw#?fzY=gr^?(C%+Li~A}6>XU8)Q8>+`Wprd|Ly0raw6k=miG|jhzrU zq){G4eM`|0UfLuMrE2FsEEp>WcH98h`)bH#PnCa$CcoV3>LOWmU(+KmP#1 zg#kZVei_Q^x}j?%0Y(5*g-mn$&AxKqTj>uk14Dzd?BbnEi9u(s*TN;K;gokHZ+@g- zJGf9l;w`Y^HS@KaNX6qLgX~Qy+cic~X2iL|;D$*2CF-WiW-jjwXT5%&Pc#y~Oi-w6 z7LWp+IwZXXRTn$u`0w%}ljYN;Z_QoZ<2^EQM`tPRXR{kjo0}qZ(D7Ff2Syqx@RPLV zZu*2OLk`r~+3SnhSwDl>+4zNu!vRQT=mQo0%sKW8g6?i1oxLX7stkaEQ}EM0`bnii zSx>iy-8aPddmqO={riXTd0zw?*&5fn7D3UvMzd`?_j>l%UhDY_(50&oTDl6--<&p3 zZv&(NflLJAMPMjw(alcPTZZJj(k7>7gURl49_KG!S8|90!whquoLV2zmzf@pWG=1> zLL^)rUbUQrMUBqH+}>#fijNcPp!Q}AfisoYTyWYyR1O6Us+zs0No_O`4nOGZQN_*K zeGO=0$WeLQ9(njv395$^$X7t@v0n@8-6W7J%$Op8^N8UHB|JcckMM|69iXC6ey!R- zMMRVpq(Hp~Z=l|zF(=ZeHM{nAGtgjqu`GL^cxyL+RdL%dZG30A^aa&Qm7RWxSo@$H zZ8V68z^efF2S{?@>}4TIj~LY?Aap?V1Kf@P)ZA4B+zqIV3;I+ShV1|_x`y&6g4fCP zN@GAlVWO(_xvWe2Po4t46TT>X`LqHcoU-%Ipz}x}EAhUt=;iU8MuWkC5g#^k3P<<7 zhUmc01Z4k5XZ_=m>v={mbIgV9k~+-v(vAalK7cDsLFsFIaOpk}CCkZx zp?DBo0{F{uS|HK^PL!H$kQD{9Ukhdc@jLxLaYcI^D+C5G@j3{DYwpQ^B`^-6M8>P&v9L+f|M zOG*1jMxW1p?a^jidwW+bdC^?mBDwoGk;^5)Ijvq(V9v1c1!wZ*H%wmv)#?Ur851SN zXwjA5-P)IVeu#S9O`bH}GgNiogSl~RVU;s5aQVw>+D0Ig_#&)l4dtOlDAC!dy8@ye z#K~re%$WgQ*Wl7R5Xy+q7SSqez0%x|_Tb!opPH$+StS`-HOHrx^x3CgB1ZrF1?nTl zPTtK~?Q=w3tHG97@sE`xN9f&ywGEB}1w~%pDKgX!<@7$Gi;w_NSqU|8Q!uR&Tw@c4 z*3NEw_wf3XBUB#wo@CobL;Hy(uY6ns)UEjEpQknehD*i&>51(aM~}czao6=k_TI_X z237}4_9?S$aLQ?;B@Sh!=5M~Y={Z5=5SMBbGN0;j^SJd95FMsnYK{Kf_B?H`u)pw& znN-g-{N|dG$h@A2uSp9BJM-77yDbx!MAIxD?|NH0R~xZgiInGU9H|5^aH5SGkJjmW z(A)0>`2_hceGye~Godol$fpj!t-17R+2NDV>i~)J9eU5~@~AUqBT8n|4eqQ1s)NoOc_@N=!`bXFh0xl`R;Py~;?;$XHj_SMD7&8QZCi^y{tY%i4vnX1L|A zF4b*mYl=6%aL-@ZuG@OZn_AtD%+WP`aDOm~BglPB4*DEBjfkzjkdfmiZ#~(3|I#4U z4TqqXdwys3Uy#7~C}4tWU5P04l@2xb3!BHy&c!c}>Pw?tjn}3dK6v$*35sDB^D3z- z1EPH%#iGFvsA*nuV%D#sM}dG*fom z-u3Q{<^e~ehardw#BiwX&J7Em6HU48ET2%R=WrblaHN^x5;C7;1i{m2YPrU{8PelD z#J}XqBIglY@M+PIk3%=-I%pFl`0i$^J0Bmsd*s)vtV6-#Frt0XRdN6Lp=i+NOAcX) z2WNvAo?w%T%#Q1?QDl9}{^?zQ9}N!|hf7?-`S;Gb^iFV$DJi7)e$emDPfe^?Bn!FV zrax7`v7?fp<8d^uc5IfZ#qFl7k8Jh{kNqW{i&PfOhd!H|8n4Nzq#E`}%o)L(UVWi- zGcvKsDOXEJU#eTaq{Y8GWMqCzH zv;juI9Axd7?BwCn`KIc<7(Llm9am2M&>RltcFbUiE-KH zC4-&h0`D{>T;k#<6G0HC!zCDVDsJ`gjBC>4-A}7rC4uRYH@WCyLlg4-CCb13vUZ4D zJ1|3cDo;`*9qyNr5mEPdsXHl>mmG;)J4;vbkBt>EiMeflXnKn+?rya5PDp=KIBH5;AJO28&zhZ9qmN)QWuM_p1&uhlW!0gBIl z_w?q*)!gC#C478t+!!N4pSrXC5w~z}FiWnX7QKC6rtwvRMp>@Iz_ZSqLDJ{aCl6AA z`Co{OT@!kZrDN*nhNaw|-8!sIigPjnMSk%*`@*_HudVJKvx+EQsU;XS=!6#kx0T zYs5i&#rVd7C~P!+O?{l9tuMGWVo^F&^8XItpKpc)hrnOlSFkqoIO3+_?v&<3q$ywp%|z*Q+pC;nCmjcp1LLWUmbGi1t}Xbu}P8M&5}9$pBe z_}pzg{-k0kB|V^~J{A3AMJ0b_vras!WK7Sp%3t5y%)MZMN=EpKscv1f>Fs$srme&5 z+k4KA7dg$%t_E`}=(7tf3_MP=9Fr7EmC>7@JIXJly^D?Ck-m*yKYl=V^!PKgxyolX zb`HMdpF5(FJ?e&o|1;{9acUq?fk)yGH~Y^2*36#;38y zB^^B}>Bu#{&K^>~!nol3D3&8!;sr=gso_sJ;0Us&ThH}!sNld41JgL7JP2ZtCpmo_ zQ-E(qH)BLY$~6?vp;-&PU9n_ihl``H^bQVbpLi&RnB6IrmmiQqG+d)VDr(g)+#5NA z4_+7Sh?=x^%|E9g(Fe{!U*s=sVTtO5lG_ufy`>*cSoOh9eNR3- z`1rjv?mN~5yom=wS{(~+N&CV4hkLvHdmBT1L5{m*K^H%(S@<#siBH~4&wu4~*luag zlDV>oksZu;av0-q%+?(gtrRj*)a)|Glp*hBZ(lrUHj^8>xt~>gwPRtIEZKbW>eNK% zJK<>BD`ZD^xXB=I0c^jS1a^J8{uKO!YBh}*t}z}zNeM@s^LcclDXo1+TrH~XhhX*ZEKJW?ee`kK?OvJph+W0UH2n;GwRs-b z2-IywD3%v^;xqUMwm#=iG-aIS6bhFRT93Shi{Qo0`Ah}V<4da$EoOVOa#U90MQB~O z1^ls`Z)L6w)$tS2u^OIdlYzfEj49iPzEgiVTtCfuCrv$g{$9Pt;xDUs-F>cZDCR6x zJ%@Zx;%}uCb*$*c!3Bxg!Ov}ZKK+(U2B?vf*sH$1vcA_o?ltZiD;OAL#q_|vU+Ohj zYag2#UJqu@C|Dibyr15pwsPT>(_EURxzyy%6+3+F!MDL;X*Jj_jKcY&y|YKXrm+_X zi=;L`KX~^dI@)ze6)qP2Y5qs&)826q-bs2%U_6=yykt#3cq{-D9&#Fgn-$NXN{>Hs zz;q|zL8O;`f-Z@6@DrIDsAuk*$g1}ESZPq{`fElhqQyq?EHydd%X5*WNg(Ur92xOO z%l#h1`q=I`rGt$U)$C#kmv|W=0sI7YIn$RG5XbY#M=9qp`yx1gZyY}uQLKY$Zi+99 z(6{ez+I_k8*_&CoEHMFlVD<{8>mkvT-MA^Wgi184bkw`RIrXEzn(jZ>qX*%QeDn(w9-jOIdZigzTY4+QjmM)ux`O|_%)j2LsUT(K^d5S$ z+7)`ETid0HLoej;T(gZ}t6B&yfHx4JA(QJEs*c6$UxLB!34gJ3n&5;cHw`Dd8Ss-1 zm}jTp1r3OhyB9bu(=-4S>R-`IXzsY3coAkg+uYo9GkD&9phcj+D$)RH@c-AbYb;nsxDF)krlf8HhM$PG`>WS3$A;D_RR0_j=X2cGf!CGiPjiS zz7&=}+_slrcMuABUl;Ay^R1xj%-2fSYvLtA?KfR=#420z*b$lP(`D(Os1BuPL@OO0 zM(Q;fo9urNhf8qAC2#65xk*{Rm2Ec>Rs+bcM1>RYzF`f%p$NoYz=g!`+-8v1-{ZrQ z{s@;ikL@_oq_Sb{{wUTEN=zqOQ$$}$8+*ZFSBFDoo zquBX239K00;u|heK#CR!mte}CDMO`3_c5^F8^6VgS0g#bw2I2(?=d&78!@kDB5!c3 zIA(Xp-mNLE+dy**=Z0)@CdgBjzm+T~*Bp?qFRcg#u9TBOK6u_?a80Y?{zx&vndrK$ z!F`P+MD;`A?w=NwnGp^zYd!8b=QhPGhvYnrPCe|{l_}3CexpJQ%a21!ycuzrA6)+6 zU@p~jv#4vc%Ap~0SX1HJQLdu7Lcex5cXb~{`ZJeF7lg;x+R2mn-I?ynyAPC=6wU{b zz`$GTI>`y*!L>)R>Fqlj7@cd2Cn`zNQ71Wh!zDnMP8o?xjs=7LmSCawiuu*epuWMF zkPOHFJG{W&ydpObk8U2Wn;z$S_+=JvSzvSsAfZEL4)uhj~{V zXXVn{P_easfy&B{w%!+94R<73a2RLbpEU~3u!)Fq(rEOrDv~Svh}1<)t9PZH1M}+3 zL|}mPeG3v<1rzA*75fF4Q-tF&ZS9Saa6t|QEu+BCdi=XG@`1lDh3wg0Y?K#4%5<4I zg*=Zmc`h^f)PUJsszrN6z|3jr4%KR4aaeId(FB+J?+0F0DsjKWa*XXZ?=M9wn$Ll1 zZ9m5W59NW)Jq*PqHLcs2q6i)cw6LjT>a`Dmu?P>=Gxilv^6zyj_elN6g+ZsTwzW>h zb#_rI>gm$rZ&=a8gm8(J2&@}RlRlJj2>gKFwVNH}6R{B}gdra(T6ens(>|&G_uMA(e_h;uh?9FeJ2=^4dlelMQ z)RP>s!ed2;HTy86se-W7!xq->TwCdb=WaPD}Q;SJvPBIdvyf&#k#n#Ex{yKWLi|L_#Vy zE3Ix~bhP0r0Cq4Rv5;JprxC*O^EX|kZ32G&5(T0VNk}R$X=-U>Bp8|T6kH34wU_BP zxZNwie+p1=EI30U-q&O1C>~}w#ROv~@lOa70 z1z)*Emhdf?f*gl5g6uVw#SlN#^tEOO>Wjn&G%EHR$odOBclQScgctJ|m5-ZB8+J6I z9;tNiy`sH9s=GU-)Z$GP| z=gl%HEb|;k=q`)=`k0fRA4QW6dDDH`rkjBhAXB=q4&|U)lh0_&w|TJq3OK9T##x?m zBAO4$XR>F6ag8r_V#kgHm^lMpu(#d7zUN8Sywc+;d!4iYWpU^w%VK2i=GN-C{r2Mv^-38ty&&t@ z8vT9ST^Y<>OD72g{)+eh#*X?-`WwAS7c40Ln7R z6Eoqr6>uskP`0Qh0FrQm6UBNwicLbLKJP0$hs*TE0Q4gzkT|*LPs?Ocv}o(j&pGZL z9v$72A(*Gq{2+BF<6F;zwd0;2yNV&DACLTQ88{1UB8|5OwNe*T^}7paAI4lsc$<}| zC31g%i^TlWcWn{Pk>2JVX*I2z)4?LRNO{~bzg@gr#3}ei=n(G6s%c&S>zGr~l=~*& zm#nm1g)(~w8*c5XdIkH`|9h3YuKkW9X3Fp2htE<&-^?iJn{=PX1A_)|ZybIG|HA4I z#{v0|?k9Z-zgko2@u6Ca(f2S^fJeZ~>h)urjWzCe#GV1(s{mGAb z^T$NuoO`}CF<*4HWtlCKGarPOaDrvMJg?AYdlk~FZ~d=`4>3&^QF}0O-@c3 zS(rZTU}^V^f?ofK!cp;-7KXwnxtjv2##xAcGjI zL!SM)j(V&;NIN*-AO!^v&hZ4Bh~lQlQ^P->&AdtpCw?Mv9@hctko-6o{!jyZ&D3W} z6LVXeMD#RAce?*SW&}&%O$qTaydcKd?|b1+JxZ904)ZX za1T2lVDb#5B+a4EV}bA212hFHbubcJfMQ|4CvpJ?0*Hb3$icha>}yXI995QritYa% zPFvz*)Us>2i1PK#Y~*+ScsWOp>JL{&^v0poLI@CP4J_R)J0=(%+7HQR?}-3F11fu8 zP^o7i;~2Ih4<%B-fm|a`9v3;=n1(`2@Zel#8*_fEbZRX(RGd%!LRsW;Pi)2`_UESG zEzigQSdyw?|&MzfiNE_}K9jQi5yv9hQD-y0}SK#L8^U|E;r+pq>} zqt>bfhD zxbh+8@1cTRUFI3H*hgBTOSa6^a+|{)ZzNy`$xWeimlrBfsh?U{p7tf2!_5JjARfgM z-4_V!}O{EgGfRLC0#&D^$DzxBjZA8d`t zGHp%G+Xnd#la0Mo&@bw;SQ?G`T6r%m(j@l7!1B6@x!jnVz@*>|qTo@E)h5-_C~vGD zoEs1$O!yanl5r;dk>=vblkj!g1DlfvdjKV$Xu3lY`uarZYd)a(Yt%WFIl@1NbCeN$; z$`hE)FOI1daeo+ib;`t?-7U{J?vB&wb*hlVAhU0FX~)I+Vo!D};kf~{O(O(S7NEZx z5{S!%K{PBlbfjmqrC?Vf*TrxeZw+315(;}k6z6h2^t$r5B~1iP1NQS;X_xswk`3xn zdIKq;oVg(>xDyL4_^?G$l&9dNkh&7CXWs>&fnR;>4Kj;E$NjYcc>i^BnmmH$*7 z4t%7#D{w3!-Ka7N@av%$4)wSJMvA}b+H*ataO7lO^niToj~DW0OY4(9ioDMI$HezF z5B|$w+hjA$K*ANU_0z*YBkx>UR%M;mVt`6JNTtAu1ZCOC-G4DuUMKPc@n6qljn33~(Z+vGD!c*y~8wc3U?Ji3IALH_hc77IQy zJf?4H{*d0nKOA$5edA_p9K zp=x!E1g`N}UxEN`PHVCD2HXP>uaZ!7eRcy5Bb?A0aLr3Na@C~X(Q%Od*#jY!iZ`0= zmr89Esq-2wzw~>+k_PwLcf%_!P6H?oR?L1kduXP(QoK`p<-*2BXiONy;Tq3maN^fU z&~>Xu*5!lY8Z1uh^dco3B78Uv-~`kEatx`zPyn%ZFQZu4Y(<8{vFeXv`{*so!2iLX zeU6vFArKN2y(dVn2MOsvlkelL6l80;>{$O1AmN=ahw4LnehW3P)tH2C4`6)`h6aB> zX2ib$^foKxc&>r&E)j@81yRNQ^j|p>7+Ut41i=LZl+9pEvH3>WHSr46N@Vk?kWoxXijwDMt{3>j-;faAlokLyyNup8EOGp z{Cij8p6Os`Avn89!AW|OWWZO@r2+gx3d^NJ7eG}M2_T1zCY}1IgJRF0VoD=QN9025@%q(cyp?nX*V zKu{!9kVd2hhLUa-5u`_?MMAne1%{CB9CB!e{_pU;-~WH_`qyHyV69m*&-2W^=kBxj zKIcGxG4lNjH&+?ydbfVKUSoGESIo5xSM(6Mx^$LkD6Rf9rdsUV zv7o1iF)xBS5By^Nf)bT!O^rpFxwytQ$KnVJw_w3Nb_*13meNiNAP8XPL@~|5kq)GN ztU!blPUAedNz5k01?s29YT#%Y!A_n|i0rJoFs@u8lab~yec=<0He_$ zP`ot91EIZDWq3GfSXP3^0QRRqAY&##SR|;Z1Q0y{3!hHkVfdT*1~z#rpwtZY`G#j| z%E~4p@kp)DwPp3SU*IdHeQqDD@f*$@m>t2SykXE^ev@3ND8q7*r*ny82OGOFkzA6e z%h-0ZTapHek)rV|B<)1n@AEg3!K?&}x61J7uOKLD2W!hq?O`T>yW0x_8L=>+oKWNv zNHzpl8ZMY#&WBEn_4+jGNl{Jbh8gH>`_B8~qG64N4E|rN$0|=YB%(aJU6ha^cQYI7 zAbBvX9*3f>A^y0N8UzqIfOxQgiU1I+zd9LKWArU~0kXMB7&#fzXYIiM)Xtm3_j>ru z_{CpozbuxqMQ!D;XBOhvN zm5eIGCnO4*V#Kd+^vI1ye4AQv!BWI+{MpP%j}^#26gYnmmCwk%BA0CVi93mdLZFvq zMiKs>bHWUCXME*kbS|rAlG^`^H6g#(z3+Z=Dh7$aZu@O=@j9&i)CiS)J8J&tC8(V1yEH%=w$Fn(|7@X7?UG^S4?Kw2u;Agk9h+cQpfgcukyLypI55v%{5#r+>Nx6+ zJ}kFQPB4rU8z}&^kl2tSUG8(6wmXDHDVBhU6vP5-&%ma01r72yh^4<2`*f7u&Xw6^ zr2cZ>LnWfCL}dxvt)!otJ>yXnPT6e$2T&a_T>x?3fi72}C@%oaMie8tvPT{#-`8^- z_w3fhMZ<_1e*J&JEICs*^Zt1M+}^&pECGPM~%UU-(OBk9jYCJ7J)z1d->;&zuIH^ z2}f7{d_(D7eBwXrzmI7ybX zBb>JnM9{yH8dK!ydcKiTB|}|QjevOr zxfL2d1RQ{(<;}&^S_iPl(%Ab~##hlYu_yPS5w-$f?{u z34kqEWRd7?A~jHk;VrQvUGzab(J)SEr1QbusXKjLtD{5(X>3f0RXj)p(Ix8Kw@bZ+ zv;fDRFYmo&p#T*i?Ps26lcL5lyVAUL%n$c`0bfmne1|a)SYALQfa{?`$}0bd%)Za! zw$=-je|k$EGAQeVJyXG3WrhPGMc8w}qJc2z&Gz7i9u2UE-67~Wj`5ZLuf6F^b+{Az z>#eGpf)oP7_LLn3;`%@Za$)nlOv=q42B9)|ll?CPvv%wHlEuo# z<-awa*<~}@Ue*QhsPV{+j<3>F!u^W&97(p0<=;iMi*{;@d+GqihLUh1Vf$QrsxNGg zKk%fvQ6bG1Ovc61d4F^${VIR+74e^7hhm-po)Jfh6H)GqxN85?$PdHZ?w0fbGA5u^ z`Szk7G`QNj+jZ#)x|c`0$>a$BUoQ3)77AiV0EOZgfPLL7 zU}!A&AP%r|F|d8RG=@T-n&2$zF9bIGN`}S)#%wCq!KjZHkcc@QI$jZ6fu6BG&I_iNd%3n8k7ArOh!gCtR_~a$8&c3K5lTZ}@Ww&UK1;EtIOy z;FP^e`qCzaIb-0;c=b8+fuwfNF1+<-Sq;0$d1aA5zPlwcdT3t()IGHuYA#)T2?D%N z128T!-Ch)M=;$pF5|yPfGwsu9FHq}ifOV`FhT&cQXXkmMXyiAvc;tV9*^hYSrmiQt z$LBhgml~>O;?Gur;te7n06)NOgzp73ME?@wOiXjdx!cWyz@!CR2+;r*wqtOtG7d_@ z1c<_QX0_pu$6!y+`0yV%d>3O$+-gsTg{8*TzxXG=Id6`y&dNw=L`}K(w#3vO`<#B~ z*y_}LkdI`cf+1i+e1-y>K;-%+;{Yu>KXkh;wqq$^#AL(nx}o*0&JIqev4m1(VaB7- z5awMd<*L`GqT4uXX1(yF`yxOEuGZ}XP9DJXF(Eo`-2s@1f#y8>`jU6`HPmbq@|9#| z#$v`O1-u4r-jIo4G9uVaF6TiA?@S0Bm5tdMzef~$&*7%UCbDD)3{w6dYK187Z}^25 z0{i^Uw9~T~4pj1Mp$x}Ek)SA82;^@7RRPuqa0*HfF$!$jdfMnQ@HZz&=X-wHOy9$V zCK`9C-H32*KIfw?>(!&qCBmbRJs=TX8Lq{zIWkH8o*pGOvSzzx?1e%Iw^|rTyP6z+ zCREZ4=+C_fYfe0?tW(+8Hq%wR@eUYlquy#ZX93?VFon1;Tz~-0wbyXZlJ3#}5Yo3+ z1xe6=0mEEx$%l1M>aM|iFSRr)qiLXINJYp9|V#i zm(TqFxK)RoNI#X&rbbf5PX{6(rhx6fGJCvbaRAbOSwgvc|4Zy@F8B!CQonjjgty5n zg#=RT#lm%v&oYw|f#auFqiQFj}17oI6M+X!Zh-45|Y-(=UO zu>x!RnW;+bsmh|+;u;Oe>7abuV*h%Xz^S{IJ>6b<=T)@GjK?VQBA#bC>1o|BZIkgu zZ@29Ude4xVh!*FanSqGZf3==Ggytuv3oae&pzmirOxUk0 zY$lKI&d|GX;;2@&i8ovbIGzzYI=gnKRSO+iDXW^lbeaq*)e=4NaanDsioun51HSiT z-m>}ZV%fP%A>XqlniSRl(E(IYOhp$QOJ$+13yufsFF~%^&orG$0N@D33qvl&W1tsm zBviK1bsAYepxl(qbJlVHf4u;uuDdREzhEl<_xW-?jqVPwO_&%nq6}NfwywXvr>Ar# z>d%eGqrw9KgEE#2#m;z_>`J`lhwJN#XqZZM=zprsvL5d5vcx>wG*^4VIac41tF=3o)9=j(cj5bcL2yw!!PaFx^3O+i3?g{QUwY_GZw0G})E<tL1dKa`|I`xq}K3M1-6VwINo0jqdAMm3ZjRUdfzVnNE!&}eF-pS|Pg?-k^}Us3ZQ zWy0wf;AuFxh8`XvT7b$0@O5>c51=!%S={cAk5VJ!dhWeJC4?M%z)9G^@aDqCi9d9L z0H|H+mmBXGkkng!J`v27?`V)e1SH*8Wz2ihA`6D4z74Hh#7dK*MeTV;4%Zx7&73x4 z+K9ZzGbZ-ND=JsA+l=jQv61t>XMFWXQXYjELqs!%xuFT za?iJGu2T+znU*J!vHp8e=6$Z!J>xpQ`OGlbJD3vc9lual5FEJFn8;V&$j!L>b9rk= z26AlBaJ@NYez?!WwWOo?_K{M&36_=%wt-6B%Ll|BdGXv(ZSFvh9)hL zd!t((N%mZ}U$>l577wqXr4HBimup|juFAzwn{w5Dz&8ttm!uH1epae`?vF~dNMh|^1gys^(g z&WJ0eA;W*ug~yK2%|JuU_9_G)E3Q1WdaiiffMDIH(q>4+rtl2c_z3@TG5!z?epgYz zk^1X?h9dc?l)*W@%@@K<^#?({HbG^(e*HFK92bKF8~`s`3nzt-`)V4NtDUfJr?Yv^tl(RkzsJm$90B}PYtvyaS2t^Pw{M1W7& zzj<6KxtCSth_TGRRbB!x#mWjg}kIesE_S1DGB8mfS;O|{GB z2w^n1RHe-BC{f`J4e^PyMYC>sjU2{K*7+L;O-?uXB<*`hTZfk!S+pxAsPbDrqc#ot zX6n?Ew&*+Lxp{}{I0_m*8@3nB4-LPd3vJ(qT^I`(#t3bObYA z)M(!`E=0u^f9K`#EFmW)T^uf3g+V0Y{k%5r`Tvvcqr@$XQ|9PWc-^Lx%i$&^ziB(`( z0gwg7V_q@=fCsXeyj#L%G7#|=rRJIgoi4UHV`t76VJu~}0{cC=p0{W6EC$xNSxQ*E zzERw_(X>Ulv1-6mMB^DNs@a=e6k#<*fdJ7%1L6cYMSRE&i<@EGpW#1Be~%VjA6pi# z8I5I}jy<>`bagEESn@KKWQqGGW5S7|Y)KCRic-AwlZYy!R9TX;1b8#bi+t+~0{Iam z>WVn;VKk{MB~HapQBSQ0ljYW0yygzDinW0Mi#fEs9*a$bj`{~S>c{?Y+9t2s6k2`M zsu!De%s?vX*K_eMTh#HL;%S(92+RJxb4>{oMCnE6eTAlcRdUHQ9!($7mQqdP5#m)y zdb)mweA9#loN}uXa%B;h>2+94bRZF`;(8Q(QUxaKb}q(GSPnxmD1l8hjLzO{%C75VA zHpZ2^_d51_wc>m{9ZBv>KOL#alIlTnDtFiB^l=}g`;MrG=s737(tQ4LV)5zMEEq>q z+*-rC(_LnUY1nKPNs!=bHC*)mUb6R{t2a%`H%O)M(VfnK(HPwgtE_dnGoEiX=`K+N zjr;loVbKBKe%RCHA)afl^YC9UGxEcRJg1zMY12I}WX{C-%B5zb8cP{75u;g`7Gmue zl+Bw$$B*vpcahOH7%Uwx+|+jkcMb8IeYn__I^$x|n^OqiG-Nc_9~3{eR_=df-!BKV z`gs>~*jj0?B=I~g`mZV^QwAio7U-=M z*J0PC7nxe=F~3<6U5>s^N|~#L7w71{UB&V@NQ)oS`WX|fcsk;tW&%2|rz^MUwuRu3 zbg2t?rtp~D+R1m9d)y<=GqZ@NXe|;sXF+zzB3+LAir{$-Dt&x*MSn5eSc-V_YFL{{ z<3-GMsXF|s`_at5BAJCxXMxp=i;xDTGv(%Nb;cL@GV5~r)r&dK!7Gih zstrCZ zABQJ=lXHM(`8=^-3cPHPSiZcPZ>-mh!U4>?&HSwe8?>N6<)A3)*Swst35uerQMPr! z_n>?a&A|5nYKZ0tZl#gt^~J#rO)t{&*-ST?P_uvHk56QvzK?9y*ljfF%OB8}_f%x7 z7jye{&a@1CCcZ^Mef-?fiW9z97taHmo8t{9}#5)1-gpc zeA%p@e*k&RX0F&3+iySCw74){F~33_{&yjh>bY!8MB75m`cQ3r;(X(K2PMyWNh-|s zgA6LDqx5sC&wmx(E|I77xBdbmSd$v zDZ|y=#~ka*IK#K}U`DR+TK0LoLCgi?D$KBg1g#`50x}Idn^IPpi_P0u&r~n)zduZt zE&N`WGeIf8Pvv>5a~`ExaPOyZ5^_`1Il1|H)pZGF>NX4_&DG4~@+&O&j8+L*sYIyg z3$pt+LI$;-&{1Bcj3j59k;A#<2Z&0@uPb|- zl^lFYusdO#vXmNCUnjeJif+)^1@j3f0Bo_5a&VaIS!)}jwKF*qYNX0{*oXs13*eBc?t zoEAO@S0-t4Is4N;r`i#92@(GC2_+1)SI^xTdL-+T`g#r*$cFXUhD$ z!@W-}P@TW{%->Zc{JoAecj#F`|W)a^N`YGopZZsn%^xzvqjY(G)dy2 z{9!15Q}y;p27WGJt6xE-q#-VQrmSuz*w+IuB+6yjQ)Fk{(zjF7#CJFCJF2E5#9!`s zJBvzDtr(2#U$Gy#6eNgtCEG5qm`u_4MP2aj^9@Pd$MSmwbX)AkT%t?-`et*1PL26` z-yX*+%mYamWyCM8AO6IZ!(~s6fQbii%7V9fD;=QTqGNxeb~R9%5_?;yfi4Lg{nfH`YKj5T<_miK1mcQK0 zIee)`d+2f6qtVIKA_g#uW&`mQUwJRwzi0v1N zJmPG;ktW=KU)60U6Pm>Ls$PzIy}mc)$pJ1wm$qQmTqlT_n-w<(TJn_LuQ}zcr-scJiKFx~)uoE}Ge48dLlWKGE1eSZX*C$a(1opAzOuJITsNNaZfUK@y>T(_;V&E4! z#i1NMBt`p;T`k6lC zAfX*$59Ykaim~N6?~;xd&3cC+()dav9($qG0JA)gZRYYxT{{!eOz1izI57ozWb?cG9v2 zMJ%9fO@nJ(yqESyV^>#Gogs2tNRm9J@gmLfch^AS{oJoM|Ne;KWK%2TyQV9Tr5h9w z{|ir2udX}$fOf#{huH7R)MCza7aipMtwMsSUt~ikK_tHy7A?W<2Uc#T0QqQ+UDey+ zO-J2avL76=*koK0gXUy&Kdb2$N351xzRMj@o2rb*(vxzYkW6NdQu==>F21V$u!H@0 z=W%T=A(j>eZpF1&-jOS;Xqv&#(_vigQuYs-Kz!z}Dn5_AXsilo9@vG5qzc$iGU4;u z=`02fhQ+||HD&DA4OQAI?GLa2P(1HpQdJrCpes1?u!$E}&RJy@w&JHqB(fW|NlP9C zg~*v&t{OP|$4|E$VM%zA?X&J>k*m5q`xP8QHxnGCgGM8~6{WPvmWmnqKIm#sF0Tb% zLe=k1Gy+I0+1gCwk8QBTZ|p`4T#h(9BD0wyd|^}X36j}JUPtJCRLZ1#I#`1kJaXrU z%cy^v@}*R87(o(fs-g&}s&`qP2*v>4yh#TA`7dkCf8C1+N z??o~s<9DA$6Dy``C-?REznVr`*0At$?%*!}pzg^(LuK|UnZ#%A{Y!{*-FxNd)S05` zLEPVi^wxK-+nz1C>|T%*pEB}wX=5^uQ-1yuvS*tD^T?wRQBp~HWf51{9;aD~=jZWN zQgKvMG3q`gdN-ooi1Wpwbg1yjK1|>CNYq9T-}lY%>{v?`^|!=RC{*lK^N|1yKL1$x zAUioPb&5_iy{d0&XXpwtJMHS8t+ZAcgN>*20XuEi=!36C|Fb31Wap1fjrL0jsBXTy zsEs~e!;iS@A~snGZE?}TIl0A)G_QmV$FulG?`!@#qgzkA)&k+%1*cE^2uvE3?gMIo zbWr9uRFM#{PXSvDu;&AqS=5{ejH|bpp9<&>>-S&P!W!<-_ygR|= z3Q6|KPm&N}plsRH8uJDUhZJ=yO2Q(wF% z&tu*PVZZjK%h=-fh8ok+E&d&F^ShD~T%};q?<4ze92)+WTlFW_I+<9k_Wd|&TKZ9S zQ7+=U(W!_0L^|~&#p9{!5c~)yBDb(Fsrwj(Fn? zp}oqwbnGYCvByzrgUWp*rVMdxo*CP4HU~P47=iK6Q4X&0?PY~IKF^FFN!}FQJi~S` zV`$(+c+{tBsVzxi`4y{Se+^9wS9`EpC*0d`D8TxKX0dJZH6nDE)3__mDuNjam7pK^;A3!*dnY!>~h@sIj*Hi zT$(@lbF^pQ^6{6BR5>NU z0z=kq56exAwu%>6{rD2|S*e!7bfvzfvV zBAiN;9Lp8#3*-meB}kz3^(haCAMB?wSXVu+2(vq|viZ_WPyMa6IO5N=s-xSAObP*X z?1mtF*Ox12H&k$u-G+!K^B;duXZf9iU@g3q6WVhf9|m&uFsOi~LYjeDyf2U6keyO> zy3%NLly19R$Twc6{TC}j5r`K{QepeHmk-x1xMuk-+nq4->4nHy^kr2G{d{SqjjcZ~ z3(dSdWGc7Krl9C~YyPT)nWcjEsw)}o1J8XEXN}7N7{^Pfs041-ly5QHb#AXsm&SEFPB7-OYq{-BYDF3mO_hFYtMPQI<1-Xxpx3FxT$W0Og!;6&!IE7+HTo#l6D@7H4nrtU1% zL+J$_gj*@(mg&gchA&97x31fs<5gS>kDeE23f6~|681*zCHReGT+T;amLl1rQ0bzW4YE>J$#4F@0GG6nuc|dWJj#50L{BC zCekj^`!1}b5jZwqw3XhLdrcUoiutBGhrZRN6M~9DLQC2)b=#f7G^*~>j_BFM^kt3~ zg|hfbwJx71T3n*hUNP#@MAA?H2}^N^wZ{O-aKgUn4D2`V&l9=wL)i*};a3st zB&^0Jl#3lTmqG#ffv%AXjrd2kTF{_OMypX0U)$YZ`3H{mXe!DgybWC}FbS-N1}!vwwxHr%G)%?Mr1 zo}DM#whX-wpjR6=lCjDDaq*a$`Rb5d_-dw>RLG~mtB>6DblOXYTYH0sMNqW~BFAC& z#CJmIUbZV_-{AKgvTukZCop06?#eSrkvDYZ4}lbSW-f=}{gWp%E`1F0(|6_j>QDk+ z`LmOA+b|# z_=^!43fVVgi{Rzb*Gk`y$y+I8l|Qsn19qI`^}fHNV;PfRc7Y1qCz{~>43Qq&cB{=VrTO9NWG&pinAYVrgADWw1W=A1U(6C{3QwF%eK0Mx(! zQcv73lzoLoS^!r=>FG|0T?3)_$KK~LB_9Y|mj|K#CckwCH-7nzxRCu8%37~ZJWKqe zt_HczvCOOFZ>5wCmh6~Z+*HWh`fzslvmQzIt}55kse5 zy+1%UGgr#+&XMlp%UyWrrL{ucJBB!YLqBnKjYm$zT8)OD~|hiNg$ZhFK92A@_^^s zKA8P7pd?r>Q%xMRVkok^T+l}I(z)s+r)tmR7ud7{3DEHsukhGLg|Ys}wh@Dj{KBDq zK+TFP2`+|4pE}DhP~E*DZl%zc|GdX@$C;BnJt-*s>l<*MxVtojJg!7x3cZhN%UJdAUQ%EjlR^Z?(RfX(t7XoNqRZ|7+ngJ~_) zt5Dl!Xw3}BWkT<4M~G)TxieD*@4VJu${d!%etan#yww;c+QClSq2yVihp0Y--r$(t zzyDNfr91*QL1CU>1vHxI@Nx|094b*4s#mbTEWujZOGYhWR}Y(<8cC8Z8HCk288FPZ zr#hVVA4CLcI4OLXxzJ7drj+tEykvqac!n`@da4T7xB_RYa$#`X>0Jo;x6AjcPKhR0 z!Sd3QYw0eFi6@!H?1Q{7CQc>z41?k}`rB}^)n_v_e$@3fpYwE0NMrUXkK%-s$P<^x z&%#;eoJqdwqE+NIuU9p_H$`t%k=IQGUJVcz}%fh@zdRl%Wb=JyROGmIYJ}AX5h^ynpnw}WLx;#aP9)s+JN#F z3Zy_fHZadqlOyf3ipt%RLgtKJKKDLBit#L9WFU8%-*KKFANG|)3}ks}b{|utWT7{o ziY%jJgHd$@)OT*X%@zO~>;JY*Vn8eND`{AJL?y zUB+rs#n4>DM4)}DucQXC{u=&fR0}S2Ory+}AhF~ctOElXbL%nv{iM&>3GbxN*)y4rc+aOu=D|5Mj2zWrAxbWH6Af5ZbbxHdQ#YZRz_8L zlCT@y_r=3Q561Y=T~_^}!_OPlh`>cOX~%eC*8Q}Lu=)$tA2+A#Dp-3DNEJ2U|hV&wN=O zW0Qf!{D!W)l-2id?&-57O2*f(N>~N2gl4A8V)`U;z7@V$;vuFNRu8_8H&cSMw!scR zhYh*Ts7?N}|BL6wD0R1?Q%fOEm4aNbyoZR>H1+ zUi4t*csqG@u1;d|AT(L(46!rU6_`<=Lu9(8pF{T`>W)7)uUj!CJh-} z_cG$eh%T5#Fv0JOkDWRDY>f4x_7?tn-IKj>jyA5&9J2cxe<-k2$_b4$ z5;vpwx_G`W##QFrtWicrN7kf=n8`Cd;`ZV7!E$--ELg3u8 zKFoeaw2{vI6-@#T$)=^^O+?C&s{?^|?BOB3jRCiJY@d9c!Vuc^{lm@FPs^Cs6D!YV zW1EuNd`l}D+o{2nY;Fv>ObK8|1?D%DITF-^bp5AcPi!2@KjuS$Ebi}yf}RhslX3^i z@==XM2EFQhmCcE*8O!dgqK_nqhd1H$HE}z^i1})Yj2rILfM~^X#zzlqN+rR{Ka2enHf#YEUIl?b zS<(Syj~c})8R{9YHz`n^zajS|I>;AcR+ z#!2|%HW)OQ2-P5zUFL9Fofga5{56^V+yJYIrW@mOryGBy^+g?-BsYD~h#pxwl3Z9l z0gsyMEH?FQ#lnVe>W6}v&${(5R|5k+mevG1V20gu&7im=S%Qne*k(bT_Zx zM>rHT$>bA@=4pB;)1E**5&k&ZODXgea#v!?F?JhB-gS>n^M8$2Vhbm+{mwcWeBqk+ zuq)mP8Juf!Nkcq@L+$e7xADe{RDs1$)<3@8%gklO*E4IryrJ432RGsOwE&p7xE;-R zL2S3|azrabR^{nHvMv_*p*|5=(KJrfKRrwG_%E6APe-6rg*-|_UhGJ7{$raa2Xq7j z)lRZzxweG{Bqk6xulteHun;1oRCkjM;9^ zSTqyC@Ru}~0=ty({9*6am5XjiiCi_GKB|w{Vh)D)gGtRTjun`pDWBuWKFSmZYPSN3 zAYy^@c%Z7`>~y)P-0oKbC{czDUA?^&Q!y%&V3-sA`XV5=hVg*Ww42d(#7cD~K_Vxe}O0K`#biSa!wn(~$8p*UaB}QuFg7 zlB;5Qm2!cV?%GUPD*^#`5txK(NXM=xvk@ zksp5NMt3(P@9}ucGg0KnL(eq1X|7skhh7z%c)p00cT>>e=|`_rsJuyDM8mJfd%fRr z?1N$)4IS9eG?kU^X?x|!Ch*QC15pa`8{Wa%(eghxMDpSbmEQ*Ib-jlD)tD(@ZApdzu^B>j)k*CUs!mccsxYzH-yS2PWB5{b|O`4?P4&)y);sR+z^W zyI#Li5!l@oBMb0+Gx{=u@$)|JRM^}ql6T(8aFlxbcd?i`bHlV^+N*ZVk((7KG(wV} zYuF&DI+PP4;3w%lf#FTgKf+()C2W+b1`f`@{~gM);Tx=#5)54Nd9&uo^GW9>uWI1= zg8!(&{rrG~NK=NSsO$NouCu2q?21{5EZmznA5VCnN(kF=Hl}B4@OC1r{bJwNbng85 z_k4B4Uadc14GfsrxS5nXL~`|B#Q9@dd0b3SfDAypOS;S5f(IY;!Q(n966u{NMd1Zo@(@!;dqk?3UK)^NfRy@4apK4qq zkzo^M{VCzzfhLzIVv=!&Y!dQ~aE(0b% z6Et%rC?#l9jfONPNIQ=5jQ3eXaL|22FyDCZV9g4vlQ7+#-pD{Sc|^{%=)!~1Acm+} zs)g|Yqh z)awrf@riOF7#CEd%lUr@&Fm$NU;036YsEt2dX>ETDNB%3tE;aAAXQjTt`hIPc=J_` zsdlwn9s&Z61RnRegOZ=RzgJFs(VqjD>diK`{qcvSO+Ui;sG_39RP^T%3JzXB3VFm& zS(68!-qttZ)mwYwCgI~K)rVzTP8M<*>k&R_vwac860$2)7Jqgy46G2j+7>YM{j;rC z_CFX39)>&RO@?l7yAm&rTsfo`-Mf2gNCVHqg#3|0t6NVjIm_`x8&~!$=7#;OsG#mN zwN>@%e^PRWL9Rd#7Q_DiwiRr5$@zm}!hITsnE|xz7zzUFYpg8dM0SWz>K57jobo4& zT73C!r`0@vf37`I$($fc*-s)o55vbu3Nv19&-kDpMQ)nVx^rG9$2sqRf}(wKy~Z;$ zUyTV)PBDE{OH#lin99Qa5)bDao&aAWz;rJCh5Ly%+^K*w&%_6765GJD>g|(^ zub7w5aT5a`CK*ZU?#gl?Qhsqz(^8o3exS=q<_XI5WkFbMtUr5p95}fg`mgTEm{!QV z&dXr5zv)5?--<`Pc%vHbg!_BVDL%j4Bx|zcZ8>WbB9kM3#>-d&>CxS_V5nf44n~I%-%c(fT(HNQ?7W?P5sZE*GKF^YokQ{}(;4?{XiC)OQkMd2 z%UgrMy*_G(V)D4h_8{eg^ix+zaW`w>RJTQiRL^p&c3Dqp{HE(0Y$P|C97h0Sv7##W z!n}+p^(1`eNoB)fMZL_I%beiRd|vOf->fTjl`<>83(M>FEWpY}+dJlRD@SWI{^Y29 zyYtps;xb`Fn0q2Y4)N%FG-PsEV5H9ECN46)p0z#RC(pN^>nZ(+Qw}y-!RKb&wm(ok z;(ZAtw0c$IU>Mz6*+26)lkU&({aSbM9sZmB$Sby+iRls=;1L>GBHn<0x)F&T%&UB(bRd7W3q7id@H9aV5M8DhAKEVk0GnBqDCgP>V zh`JH7%A{MzYu@DKGO17Zb0C1KN7l?x=b* zU4MG2wCWf?a~OtEW^c!*vN&uFr_+xP#T=js=+}94Qn3D}nmP-pUUWDX!)4|z1sFGX zLSp~<!d)STj zqpNyJb>-5JRB{2*H}D6m$pV(;qU}vwnERA^%8ht`A0|tz{7LNbDVD8HaG0!?$pquF znEc8~AiM#^oZX^k0bsiKAGF4FrfD+NpPU2>TSIUgG>8^%BM9wlEeMRg4BIJ%blVw) z^aDTj%IoXrxPNzczkI~7@9; zr*Qz|vrmp6Zx=cn>N*%5KRNzI%UEsEA!{B7Lo31LJcY_3k41Zo%W%}F?Ln#gWG<&) zwg|g@9xD~oV4z{yQW*X-O^^)dFBG2XYWdedGT=_}YxnWoV<(ebq`3;S99cyDYwi2( zxnc{6(KH_qlB+DMIHv7;YLwp(jK5Czcualc>!RWHs(&@NWpf^12s1eun>b)DdUee%tdmTC%slNuI~zcI*pL2o zpf+$I^-(x~plb#W!i!3!OJPk!i$=Lb?3y_7Tl8ZV5j>jkjJzM?dv$*PG#ZKZ=yN;* z15bnx1D#}pC#kXzB9&#wM#z^i(=gR*Ja}?`%iyposdv<12>+bDBE56Rq`);JJ44PV zjc_%VvZQ)~mO-2BDPMS!>+E5Yxr{9Q^7D*dZ4!P@Yx40kuEI0h!!H1lMbM@g_C412uU&8O#+&yX5-l z5KpjIA&vA#YgGBf+ZR-fyHJQdsWKf>jK3Gebj|;6Qfo@*$?R?i>2BiYTfoc@L2_^g{_~`#%=r~iy2ksw{@1QL!FS&D_@wf7LMCTY&7P0Q-h-HAW6YOr z>U)}qLcDnW;Os@hI|FIMMa6GNj(0JY`h!Z4Y&d_vF;2)>zcSrLC!19=$63m~zr=Il zwqIYda1~6>_C$(-XFOfG$>e3XbkFlxh~M4q7|z!&p&n%iic;cYfu*~T|E7#nqdgk) zpmK`ddduZbC}UuvhAolT0X(=W(CJ{0zDzRrnmk2k}`RUVD5kdLLAH5lC>E z-NlZedi~s;|KN~c^WKLZ`OEf%gEMTQv6d;W9_GYFC(#!#m81Dh9}eE}`aVpxG}5bR zIvbMIc*6fkQcb-18~TrqLn@rO`_lq(h7V?YDT&)&@82+NccP#B)~g0TMG6^K-5CPy zkj|uwO^hOo^Si!^;J$LfG7!S}P=z&c#&I4)H|mI)@T*t;@vY$AGw!Am13Zdm`8VQ~})E$BZe?L`Js>IifHAk<`qoWNJB$K21$L)lyx%_A3 zhrAr;H1l0x-6xIn<+Jrz;Oq`$)cWLDj%E&qj*(jyP6ie)HjjfjO3gkMV8Rn77(g}@ zdPW;n<&OFx?6xhYcIfZy6*PnEwmrWj0 zO_sdD^)kU5UK_PXyENc(drkrYi|^xU3))R%NL^2wtHow-`79a>Rd%oZ`iF=zEL$iA~A_5U#S9pF&F|Nm#6 zO?I~I6)JlZvPp&PkP4BJbk}atNRfvAP7%wWnzYwA(xhw2nq3)EiXohk>L`TwSt(xh^N0UE#j7<-)yU&P z1nCC;2bY%Dte%CNZKO)d|tz}dA?f_0EgK>v9H{Z7~{h-I)YF$ z=1bAhU);R3g?ZCJ()@=)`;HR1=81%m3LUwhHtPF^=kKq zG~Hr+{BRUXTHbc8;L!0w4Ws)#2Mq2F^g*z9%0gpQ$6HI>I14{F*vnDs*E z(pvqmNl&)BLr3{@P+oiT+Pq%p0=WBkpwex2lma|^^E1mH`4{g8sjMe1qv)qKcg8zB z%{0F zQPQZEnE#DTC4HOvH^F_sgG{os~m#eBxc6tiN)L9JuOq=79^A=w-VcHCWNxkC0kbqS7XQ$0^@p~o|7 zr|(GOn&8{>mNsL+uSZy`9kKMKA^cvL-L30UsZYSe8kDKgp z?zF&yDRS=3h{bncEGcSg6Q>-vIK2mB)N=E2&=OY*qo61!f{9P!$T*pbXGs?mRtEx|fJrtp{op6^-4?-KK zc(Lch`H(;(RW^W@ci%(DIKSN%_xeH1_>hoCn2q>biIPUFud{p0;(w&19?TQUz@VkwXNXt@@?{XVa7-DklD@?m(`G(nMyRqHFpEwaGTroVCx|S(HrF&RJL|U+ELE#`du^|@%NKCThn$_DB_H)%fPWvf4Ym`$6#bBoH_ zRbXW*IJg}J3k-X69G;r7yP$-#kL#zuv;9>qX-+V3^1dmIZ^?o7ntR*Ln$C6C?)p?6 z9PT6U9_6#&552q&k0EXWGcy~P7YFdYNww^7m9s?%Oe5X}2oZ6re7uN?TrBl@sDQy% zQn2+Bs+6b72k`9u)w{&#_iI`^6noQN)(Mu`KRHJ4s@iMaxJ%|qefBp3cQG(qNjJ+a zP@T12;Y^Kk0CEo-CXA2&rf`yV;^+Id++;j3|X zd*s@rwH^xzbdBRrSl^MSO6|DQ5KfdC~node+4FnyEAojwKHXM+q{ zqo|mTkct?EazMq(VhR%!UJ*ia+=7Vd#|u%@c`R-dhdB2`e|)9?-Vca|lovn0<}5#GQ~ffoKFFmtj@; zH2O01g>ox-7Sa@o!bFdQ&4`d=KV!e_e|{uH7gDZ34sgW?j~ZWYUW48@8j&E6w_=U| zY(I(BnvdE%?sqA(CK>eCRV3`F5a3k_Em#x#u3H&8&|SXD0Z$Fdxu7S3g(oiuxlZE_ zeqI=V{kBpvL!&p39@4T9&I$@A#(~MghgdL8G4Dv@B_UWCmsCC}JmmFH73MH;68kFG zU2ptUV(COg?JPskx4fqB72*>c9vOnlzd|`0HlAk*u4ADvlGNv(xwY(Gp8J@EUm5HRSk}Fh&ekiut0sq|t{ESQ*FE zM+jKp$VZDR!Jk-%0F}V*U3^n=DYIJa52k<987LkO(Z7HG zo!h)oSsxfSp~vtMKjq*-q*9~1i1UvdQ#;#NY`)9~XLR)Qc3wdN88_O50uVV;X0 z>=t=>Bfg|aWZNEe&tgToi-8-RMYAee^5>4+u7lUyM*>RZdNSi%HovR|d1~$CK=#ls zrTF(LdE@!7>V_+Ql!8-VDcmB}E-DJ`*I9>r{AYC&Z}*UX<|U8L*vJZv#=VA#>xf!T z#H*2Vf_PbncX^RQR=fDl7ZWHG4Hd}MG+nFVHOi{nuW$&2{~3Y zA0!`L&W$O_pAZ0YG;|kxX`|r*KH&15?cPQ@`k*CHExgD!bVZ?8f7YpPlyzu{g`goe<`JHC-tUhyE zGI1s-JDO^W3fig>xpm|j`e*i;oitkQj_f=ndo<`$WFN;J4{yPG=;vxB`P+SK)Ofs`N|rayw|{Fs2Vn+mZmAXQqYni4|Z|@;|2DB`{I#0=H?ya(@eUlLg+jfJyywWvXKQpSj4TCN%CN_z&k*_N6IH+6`Lp>BDRYUxXwCRlzgrOBhKlG> zfCmo*)k!%79kK@8hpN=d;kxND*A2ll15~5GXslEAAfR8OBR}(^Y+aAN??zhJ?Y;6X zuV!DczBQ@C30GiV@W|?{_xxy-CA62r07*(Z&~)h36qB9MBY#>*1_aD=pR(3(AS;u8Gzjp82aLXQXUHvWQr+cpQ z@V9iv<9@bIZlSX!QZk0~M>)T4bFD?a3&MOBDfo;2vSH=7IKy`oFU3B~`wb60DvM5} zeo{hj&#aQ0PmXT;d4SSr5a}59r`Q)=3xOwr+Dr)t z$}9jWk$70BY=gc0W3h5A6y8MvWPnWn(0cc{dojnM!aeZw1eDX0xY8N~#+|QqDu9is zH~47wGymrq0U?P$L5Qb2j*Q*~0_7$Zi3Gc(6~}C)@YiB2qWH*T+{&A|+VKy({)Ikz z)Q#sf{uwwEkMl$0!7b_+dD%hQ@_gm*!*pBQ&5(>liW`<+4(Mpoix@0pITODlw=dsb*M0HY=-K1p8p`AqgZLHW z)0Vslj8OSjC`Osy{f9|T>9TdD&xGuaeB-X9TBx}1D+$D(qm8S2YH1-&_HSAAk3P&q z&TFBfSuJa0lf3TXwzYf1hOO%p>D?u}Zwr^y=llCh4Zq{O9N5_&-4E;g@Ot?;g9aC+ zK&|a$)%i1*q417QzLVD?^LtanhGpqfKki5|V*F-Dp{i> z?hJ4;5>_Cb67PbojNq#83;}g>6$kTJrEV7&M&D(?h-{OT#LA}Q?kH~pjKS&BOPB%k z@q}JeZ*P9mw#E$SEKl^I&D3?^3>+o?D9>CRPLr&fXi0Q_Z4?m8iySvxeFkJroU`}6 z1ej8esl(Bx-WikOz{~axKDoMw+BmK9DEGahdy=Ko1~0RCOFW-|!>>}8My2!^`>b&v zuiOBz^BIdvfoi(dmY3WdS9gy6)*?k%a9Y?toit`tX_H2>|9)d| zPqbbtGeRqBN7=!cqcqXf7yY-HI@(J~9j>Uv#Y8&4~70 z6Swkm5?rPf=hv)3uLwuQYi>av1|8o=clhtMQkVapx1jE`3{~G^{4JKiCwuC z{@9fHVaB$5rQ^igh^2;KTaVKSau^0`m7pt61-!UZGHy8QZS{!X#HS*5TSH)+_`Bw+ z3Jjnr1L=auSsx2{=f{4DAL>oJdfQ)v=6%Jz0{}E$5M(?sOOL$;4U#sI6DG%}Da5Sl zGW90`mM-(!We_N2VcAowih{Y*yC(LIN2ATLUSuQ>YJLTXZ~iFOXZ<0z;RX3s@R_rw zYUMEO+Yv>Rx%umNQ)dz411V|#jbvTp)_o_|OE2#>aP*R^&2votQZ;mL%oPE~kgG}t zZN1_!Sp53jlNul0*@(`^$D~pH=XGd4TE%$c!o0V|d(*wOiqgMEBT!Q^K~+Q7!MtGF6)hWAurQ#{=HxY7s3M=a{Vho`(nam zHf0bu9Xkk3;jo9jtfx>oy=B%ysve4&vAfVcV4o&B{oDnTfKYR$vi3DE{aCC-)*nb1 zl}zWO|M1nnPq+EYTd$9fw%FmagL+$kUvbW2i*UtwkK<$I`CQ{R*`CMK;nX|uYCxzV zKPlhqu_ISk(;QTxhAuJG?uEaotjp|LfGcb0qp1DioTNEI%!a*1A-tlyqmd-@jKdd$iY9XMfFx=c zLn@gm{GK>)z)yZ>OV(PRj&WJXOG!F23=sk<&^`_v!eD2xWZCkHodiRzDUl|mRWYmW zaSW*HJF+ne*oE1b0)foFlj9m$ZaW;Jb9GI4dm4Db#GH%!ZJFGn)AbSZk4H4v7+$~Y zwtSG84}sO*JOj2UT_igVa{K`0gm-orC3k)_6nF%YX*T&cS6F$fSkIjhuiSJOepaTo z`01^}VyuiTb`y5CGGd6NN)K$p{*Pv^>I}#Ua9QthVymbE))rL~yoEI3Xq7rDT4nWp zhuQCPG7dP`i_+$ctdKY5&y2ErdNbskw{kdUUisb8J$}nWZ^jR?Z-^uLs299XJos{} zmZHx%MhWO;@G3FHydoF@M5)L2cS*e6QBJr+zvO|a7Fn0czeAh%51gDt4~BN)7Dn|d zPygL&)Zmd(7K7-VdqcQ8ceIlWeTI2WVPuAD)JEHM;tqq48wa8zo-Cy>iVYI_ciK+Y z4j7uIC{UM59C9ieQGhZ!l=;{-HSf1e*5{ATw=cp<2#W0!S zOf~)$lQEUypyDLJ~ zqPVhRO=Nc-g_^!E?Z#l=qPtHK&2ozn@$VwL9O#N42;T489A=VG-en+)4vrLXntCl{PC}Xq`zE+&UAv0amD0JUBl}-XYa+E z9M8o2HkJTsqR`qKzkbs>7;`ZBAv=&$+oXAr&E{Q&%h+;-;jctXkF6wk=39U{19rhf zX8||OC%_ELqVtVb@$t?IVS9@}j*;_@ADWGtj;T1i#Bze>A??YK?uCVp?+@n{n`l-~Wq@CO<(rG7o_W8wy)Uqf$)82V+c^2O8n&TmBD*50c zu{49qxKtj{eKOK>V53t-pBw7`e$^>?H`G7D5EEM9n+^*gg-+44w=WV@^vPUHAe4L( zfY$WegJohQsuZfyQfVqZLA^V2vAOf|GgLS z!7e>uXXoZq+cQfbk7^V60$l8?D?^%LQ*5gD_`7yU4lWPZ#rLMAUMFZi0$-l4k}Yh) zf$J^czy~<%qLv_2CE&n{PU_XQzN+$=Dj-Hu4vP8&jEKAEJ4iKnl*?ibqnL55f5a^H z-@6xZA3IzO6G>)04iFiE&RJod%Voe72esg$Ok6fvJYHX4-~UgM)QmJpj>7HZ{PZMg z42~ZaF>n7`i@$Cp&D_Ut%n;Qf^6uPH7LiwHsomiKQQ6Gqw;l#r$V!O?+ zd~Ff(>^z%)1rXUL_e1@%vKB)_HylC+t6;6=Tt+$X8}5RRox&DTdyUpX-t55>JiUWk z9GC@KThF&Cr&~v2^ToyS6c=~jy9>Yn@jAqHwUO+tK`O$#4bh>>^zZYdw^M*ENQE4Q zshRi*YZY0H#Ns;LvCMr_l7+kb-tKte+P5MLZ{_pGpUPsk|2!a7ziNyV%Cp;lGVlmp zkj3t5;IN`@h=IP*wQF+0e?u?_x2u0jxk~#CZP++G@+?|7>lS}m&Wuou^el!Gu(g%5wb^RouhDR2)Kk__ zann}tE^VF_0Q{t{lS3+8CtFdYhuQH`Qm`>LQe+%&kEHE??U(-r`d7iX z`EFo4)5>B45qPk0)$R)1-4LACWPOABsh*J-x!&ZR?7V3cvap~2ib{G`4UX&+i_=RB)T7`(n%(dD~i@a0mmK51?jH3;#o zxqvh{tXsKvJJ+f5xj~2@|p99Wu+92O#IjW zm@+k0byAhfUnG#3twue#1Bc>Ns*^hvn!L%V#uWMSxJ2xg75Voe%L4yf!WCiP#_?wl zMN%>;k(*}%+D@}LD-(QZT)e(a%~d465YcZn!`97i<-_Cak?aQwZxEQihTVk> zv!ci2gJVCXfe^>dD6#b~-uG7-ZO`rJV7`)9WG=xiIbz|MDU_T2NNUOyk5%NH3_@ym zT?RGnpUV05X%Jx!v*nIml9#+fYu+ZVLNaG72rKPVCy72I;52hs4t`4zpJT;&Lp>3W=U&zaRpcgytH^Ir^BQiw72=TuClTeMTP0oN)z z^GW!q2UItLn2BHaNhxSWG_Tt0Z-Ab*+-@d*^QCm3Q-+w8@ta=+BZkthFOKv* z3Uhqgb#>De#qIrz-n(ZqJ+{|anzWmDB_|{oTQVtw#w)J$dKQ7>9Cu^yN6&8{9bfDl z?^Vo|U7Grx=WYcRalkXM2bz62h+Pyp{`0>)S^x)2XhFgUaB5*_D?Kvsm(Jx&yBUhT zb+uD1RPnL!ZD%kmT?JXODWi&o_PA9ffW%JH=)VLY zwKO;h$ny{nVu#K528!{E^~v{Vdld^&#hMT>XpPXw?z3k&M!S2Qr<1UA!B=bhgDt{V z1%E{@tjgD^NRd-JC)qTU0!Swx^eUyks7@0RsJB80^%(hFPdzpcr^&h;p^j;p_!%2y z(Y4w6&Hs!UAGxZT{U|*7G@9M<;PSf9jy^i;93@}a?tj;``AK2%Cn4@@xRscH2%0R- z?e`-p_bQ4Y(X;nCMt;-rCdvH^@%A1T@3BmwA97@Tn8vVYUSvZ5bf^{LP%;Zy;hpbp z(E(bgt@a35{ilSZeB}A}NIEjRiHXjMzjCM18YCpOH;6w}>_hIE5RLm<30l=BSVb`6 zw1BJC>&MFnDc!$PQ;rCMNbZ0v$wA>VK2nT)tC1js>Lx6zgBHkQq;B0&N zaow%gL#lnY^j5t{WYM$dnyoI6dg`<)?#~XahU`tw1xFpEOSHx{HZF#ZmpokOP|Wo4 zmWz%JL|U{Fe412hH$Clsnb~L3Mo@h!nj*cg0hv6)>8stU(z$ev3MLCn4Ecz=y02b_Qn#NH*1ks1ks2|C7?B zgaf3|8Nx1wJgUY44Q3~oS?Em;gX77b*fMJxRifj~7MjHZ^P_hWA9gs>0kNu{D4;-_ zF=hJTw0_XwWqBC;nx%JX-r5FjqTiK(m+c%El{YFp)nQ#$NV+bPjkv zAPQ;>6oV!-pnI$k>)Rm4!CB`A@HbOtg63M1|3S3lqIlJ(^ccU56a)x4s|;=-jl!oI1#M=E4}%xqOLgYWy4 z0r{%{8EB$GwWEhKiSpKtE9H4UHr{XU->V|06B)lT4kth?%6xn;*8QtMs&nDto{(&E z?;>@(6jjoXSy~D7{SRIpn;SMRwAZa*C+KIcFS_qVu`3_R^Z&aOsbo{bPA9T?KDkbu z<5m#LJT6L+$~U8JR<1?r8b`FdHNM>`?W@dhM#a85P{ZiTN~w>LQz-8oe1C9hCVy&U zCLy&h^=!}~(v|uy+A8=L0khbzTf|(t-+IIuzRAZ_2x`s`d}j1tx`9^3lb$zXd~r!M z=tb6S{$LGR*hUU&Ci;T>Aq;1$QJ=Oed5+}EN!#7|;3Y;04>pa#l(^~C{e`r>LJ=fz ztZj?EI+mNRy&f{=H#_7rZT4UCa)7`bAs{Q*dn^-8!SLAp^lm`QHUhKkKS4Do5agl7 z@6Fn|`JM%Fpb*F(!e3AlSW2U}wYY9C?GGAWV&xN~@;Ux5XC`WoBJ}t}8)u zTG#~w1BZ)viXIDz&!KDUfqF~*hh7F^&jEMC(*C4&8)-^NdFp-R<914e?aP^yxPm#Q zO3UB*Ki?3~I)Sth+UiFTh-1<$BTaU(vUw56#gXoqv0~n#-Y_^e6L4h^l-e&N#(CJ0cPq&rv%0;s0ob%d~M<kmSRO94Mm6@gt?tA zZ$(47DFY(yNBt+X6 z<8916sK14$Lh?3JOkxYc&?{iU^&@oMzgDH^G;fdhR5Vta(-JBWEC_Y)r4&2aIvGw> zDBKv>hLAM)?pdrI=aip4O)POmeCtQJsZh;{-8=>dSDuh};4_dh@Sk$Ba1XaXCvM&# zUtA)OTmPd_$k}R)yK;lS;#^AiP)W^qOGj+1rO5x*?+e@|FVPOuQyoSVKw}>6gm`@Q zYMuBkuG~k60j7KHDco5*!MXpe`7O_-lFCo~Va7k^3>)_@ZA_1^ z4GeCHHBokKzO&gCL@S=b`xcXs{_1WWP_eQ4_{g4#YEIO)JX zX@+DI><%&gx548o2t>n&jMK=aA;U&HhV-k2S1FRG z@h-A>*sK#1mf_vjRziBm$!p8YbWd$)Zt`qsB$&*E>#fw@&||Y0$sHt&tneIjWOg@d zJaYkU9Zk2#c{&y+;)>V58DVGoE~GQTar6x%F!V!#2?{Uco40OLwK=N51lkiPk9y*} z4kdJBB~vomf?LXo|8A+1DY?T6SnPN{l-PP^6OI!Qy;4Q^g=X&Ll$Qto=D26}Jy%Us z!|cZn(!bH~pB$8^R1YflrARkRyy4W?APB>n{m_Hihmu(`4I45ZsTcMSmpAZJED~x< zwH+q+Z2E}PWm0b7VM2ynq#J5q-^t6lU-v~P6r|J%swZ4K4nYK z`Y${?-D$lP_qFSMtyM;94P+(`cn9Qy^$i{d=r-W;0}kBPaROQs4S<-y;>OlIKuucx z9ak2?%?hj=PO7$-j(%5iCO@t0M)D@_kyT4nsdZKQ*=~fU*6H)f*_#CSCXaWTK5$=i z_jE-}D+IDPcxiY-lqNZ*QgE=RRTl3|ERtOWIS%rbNqE?F679kU6k{ahslmomp7Hmc zM;A^NZ*t8L`w6oqT0+!V+MENvf;Y>5`s7bKrI^Bp%*-fPuI&Et(DzzF7&u+z+-}+# zlwY_RUzuyO0Gonx0-mQs9;PE!7n}%khSlX0__tY8h6tN>#@p8Bx3at^; z52>x4IodoY3glQ0eQkP9{r$)DEwrhodEdpIe|_|SnBSa9s+#|KG=+pn@57gZY%q1*!$xTlRRGlaWAgB+u_^jyM+ zQ^u-&A+&9Oq*CA;eM>eAvI-lL8rzWi2s27|0o z?&^f

-Flyte is a production-grade, container-native, type-safe workflow and pipelines platform optimized for large scale processing and machine learning written in Golang + Flyte is a workflow automation platform for complex, mission-critical data and ML processes at scale

@@ -61,11 +61,19 @@ Flyte is a production-grade, container-native, type-safe workfl Flyte is a structured programming and distributed processing platform that enables highly concurrent, scalable and maintainable workflows for `Machine Learning` and `Data Processing`. It is a fabric that connects disparate computation backends using a type safe data dependency graph. It records all changes to a pipeline, making it possible to rewind time. It also stores a history of all executions and provides an intuitive UI, CLI and REST/gRPC API to interact with the computation. -Flyte is more than a workflow engine -- it provides `workflow` as a core concept and a single unit of execution called `task` as a top level concept. Multiple tasks arranged in a data +Flyte is more than a workflow engine -- it uses a `workflow` as a core concept and a `task` (a single unit of execution) as a top level concept. Multiple tasks arranged in a data producer-consumer order create a workflow. `Workflows` and `Tasks` can be written in any language, with out of the box support for [Python](https://github.com/flyteorg/flytekit), [Java and Scala](https://github.com/spotify/flytekit-java). + +## ⏳ Five Reasons to Use Flyte +- Kubernetes-Native Workflow Automation Platform +- Ergonomic SDK's in Python, Java & Scala +- Versioned & Auditable +- Reproducible Pipelines +- Strong Data Typing +

🚀 Quick Start @@ -80,7 +88,7 @@ With [docker installed](https://docs.docker.com/get-docker/), run the following This creates a local Flyte sandbox. Once the sandbox is ready, you should see the following message: `Flyte is ready! Flyte UI is available at http://localhost:30081/console`. -Go ahead and visit http://localhost:30081/console to view the Flyte dashboard. +Visit http://localhost:30081/console to view the Flyte dashboard. Here's a quick visual tour of the console. @@ -104,18 +112,22 @@ To dig deeper into Flyte, refer to the [Documentation](https://docs.flyte.org/en - Used at _Scale_ in production by **500+** users at Lyft with more than **1 million** executions and **40+ million** container executions per month -- Enables **collaboration across your organization**, as in: - - Execute distributed data pipelines/workflows - - Reuse tasks across projects, users, and workflows - - Backtrace to a specified workflow - - Compare results of training workflows over time and across pipelines - - Share workflows and tasks across your teams -- **[Quick registration](https://docs.flyte.org/projects/cookbook/en/latest/tutorial.html)** -- start locally and scale to the cloud instantly +- A data aware platform +- Enables **collaboration across your organization** by: + - Executing distributed data pipelines/workflows + - Reusing tasks across projects, users, and workflows + - Making it easy to stitch together workflows from different teams and domain experts + - Backtracing to a specified workflow + - Comparing results of training workflows over time and across pipelines + - Sharing workflows and tasks across your teams + - Simplifying the complexity of multi-step, multi-owner workflows +- **[Quick registration](https://docs.flyte.org/en/latest/getting_started.html)** -- start locally and scale to the cloud instantly - **Centralized Inventory** constituting Tasks, Workflows and Executions - **gRPC / REST** interface to define and execute tasks and workflows -- **Type safe** construction of pipelines -- each task has an interface which is characterized by its input and output; thus, illegal construction of pipelines fails during declaration rather than at runtime -- Supports multiple **[data types](https://docs.flyte.org/projects/cookbook/en/latest/core.html)** for machine learning and data processing pipelines, such as Blobs (images, arbitrary files), Directories, Schema (columnar structured data), collections, maps etc. +- **Type safe** construction of pipelines -- each task has an interface which is characterized by its input and output, so illegal construction of pipelines fails during declaration rather than at runtime +- Supports multiple **[data types](https://docs.flyte.org/projects/cookbook/en/latest/auto/type_system/index.html)** for machine learning and data processing pipelines, such as Blobs (images, arbitrary files), Directories, Schema (columnar structured data), collections, maps, etc. - Memoization and Lineage tracking +- Provides logging and observability - Workflow features: - Start with one task, convert to a pipeline, attach **[multiple schedules](https://docs.flyte.org/projects/cookbook/en/latest/auto/deployment/workflow/lp_schedules.html)**, trigger using a programmatic API, or on-demand - Parallel step execution @@ -133,19 +145,21 @@ To dig deeper into Flyte, refer to the [Documentation](https://docs.flyte.org/en - Declarative pipelines - **Multi cloud support** (AWS, GCP and others) - Extensible core, modularized, and deep observability +- No single point of failure and is resilient by design - Automated notifications to Slack, Email, and Pagerduty - [Multi K8s cluster support](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/pod/index.html) - Out of the box support to run **[Spark jobs on K8s](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/k8s_spark/index.html)**, **[Hive queries](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/index.html)**, etc. - Snappy Console - Python CLI and Golang CLI (flytectl) - Written in **Golang** and optimized for large running jobs' performance +- [Grafana templates](https://grafana.com/orgs/flyte) (user/system observability) ### In Progress -- Grafana templates (user/system observability) -- Helm chart for Flyte -- Performance optimization -- Flink-K8s +- Helm chart for Flyte (coming soon - June) +- Flink-K8s (coming soon - June) +- One click deploy to AWS +- Reactive pipelines & Events ## 🔌 Available Plugins @@ -158,15 +172,14 @@ To dig deeper into Flyte, refer to the [Documentation](https://docs.flyte.org/en - [Qubole Hive](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/external_services/hive/index.html) - Presto Queries - Distributed Pytorch (K8s Native) -- [Pytorch Operator](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/kfpytorch/index.html) -- Sagemaker([builtin algorithms](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/aws/sagemaker_training/sagemaker_builtin_algo_training.html) & [custom models](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/aws/sagemaker_training/sagemaker_custom_training.html)) -- Distributed Tensorflow (K8s Native) - [TFOperator](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/kftensorflow/index.html) +- Sagemaker ([builtin algorithms](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/aws/sagemaker_training/sagemaker_builtin_algo_training.html) & [custom models](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/aws/sagemaker_training/sagemaker_custom_training.html)) +- Distributed Tensorflow (K8s Native) -- [TFOperator](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/kubernetes/kftensorflow/index.html) - Papermill notebook execution ([Python](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/flytekit_plugins/papermilltasks/index.html) and Spark) -- Type safe and data checking for Pandas dataframe using Pandera - -### In Queue +- [Type safe and data checking for Pandas dataframe](https://docs.flyte.org/projects/cookbook/en/latest/auto/integrations/flytekit_plugins/pandera/index.html) using Pandera +- Versioned datastores using DoltHub and Dolt +- Use SQLAlchemy to query any relational database +- Build your own plugins that use library containers -- Reactive pipelines -- A lot more integrations!

@@ -202,16 +215,18 @@ To dig deeper into Flyte, refer to the [Documentation](https://docs.flyte.org/en

-Here are the resources that would help you get a better understanding of Flyte. +Here are some resources to help you learn more about Flyte. ### Communication Channels - [Slack Org](https://forms.gle/UVuek9WfBoweiqcJA) -- [Email list](https://groups.google.com/a/flyte.org/g/users) +- [Email list](https://groups.google.com/u/0/a/flyte.org/g/users) +- [Twitter](https://twitter.com/flyteorg) +- [LinkedIn Discussion Group](https://www.linkedin.com/groups/13962256/) ### Biweekly Community Sync -- 📣 **Flyte OSS Community Sync** happens every alternate Tuesday, 9am-10am PDT ([Checkout the events calendar & subscribe](https://calendar.google.com/calendar/embed?src=admin%40flyte.org&ctz=America%2FLos_Angeles)). Here's the [zoom link](https://us04web.zoom.us/j/71298741279?pwd=TDR1RUppQmxGaDRFdzBOa2lHN1dsZz09). +- 📣 **Flyte OSS Community Sync** happens every other Tuesday, 9am-10am PDT ([Checkout the events calendar](https://calendar.google.com/calendar/embed?src=admin%40flyte.org&ctz=America%2FLos_Angeles)). Here's the [zoom link](https://us04web.zoom.us/j/71298741279?pwd=TDR1RUppQmxGaDRFdzBOa2lHN1dsZz09). - Meeting notes and backlog of topics are captured in [doc](https://docs.google.com/document/d/1Jb6eOPOzvTaHjtPEVy7OR2O5qK1MhEs3vv56DX2dacM/edit#heading=h.c5ha25xc546e). - If you'd like to revisit any community sync meeting that has happened, you can access the [video recordings](https://www.youtube.com/channel/UCNduEoLOToNo3nFVly-vUTQ). @@ -228,7 +243,7 @@ Here are the resources that would help you get a better understanding of Flyte. ### Blog Posts -[Blog site](https://blog.flyte.org/) +[Flyte blog site](https://blog.flyte.org/) ### Podcasts From 3abc69a8d483fd8585f3ff406a427aca55036333 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 12 May 2021 12:55:07 -0700 Subject: [PATCH 75/92] Rename admin server config section Signed-off-by: Haytham Abuelfutuh --- helm/templates/admin/configmap.yaml | 2 +- helm/templates/admin/deployment.yaml | 7 +++++++ helm/values-eks.yaml | 2 +- helm/values-sandbox.yaml | 26 +++++++------------------- helm/values.yaml | 2 +- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/helm/templates/admin/configmap.yaml b/helm/templates/admin/configmap.yaml index d8268a5a2e..504f30c717 100644 --- a/helm/templates/admin/configmap.yaml +++ b/helm/templates/admin/configmap.yaml @@ -14,7 +14,7 @@ data: {{- with .Values.configmap.logger }} logger.yaml: | {{ toYaml . | nindent 4 }} {{- end }} -{{- with .Values.configmap.server }} +{{- with .Values.configmap.adminServer }} server.yaml: | {{ toYaml . | nindent 4 }} {{- end }} {{- with .Values.configmap.remoteData }} diff --git a/helm/templates/admin/deployment.yaml b/helm/templates/admin/deployment.yaml index da644f9731..c6cae03152 100644 --- a/helm/templates/admin/deployment.yaml +++ b/helm/templates/admin/deployment.yaml @@ -1,3 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 2f270f01a0..80bfc94513 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -226,7 +226,7 @@ db: # configmap: - server: + adminServer: server: httpPort: 8088 grpcPort: 8089 diff --git a/helm/values-sandbox.yaml b/helm/values-sandbox.yaml index c7304b037f..8d7ca4d0f7 100644 --- a/helm/values-sandbox.yaml +++ b/helm/values-sandbox.yaml @@ -1,4 +1,3 @@ - # --------------------------------------------------------------------- # Core System settings # This section consists of Core components of Flyte and their deployment @@ -30,7 +29,6 @@ flytepropeller: {} flyteconsole: {} - # ---------------------------------------------- # Sandbox Configuration # Sandbox allows to run flyte without any cloud dependencies and can be run even locally on your laptop. @@ -73,16 +71,15 @@ contour: nodePorts: http: 30081 - # # KUBERNETES DASHBOARD # kubernetes-dashboard: enabled: true extraArgs: - - --enable-skip-login - - --enable-insecure-login - - --disable-settings-authorizer + - --enable-skip-login + - --enable-insecure-login + - --disable-settings-authorizer protocolHttp: true service: type: NodePort @@ -102,7 +99,6 @@ common: enabled: false flyteNamespaceTemplate: {} - # ----------------------------------------------------- # Core dependencies that should be configured for Flyte to work on any platform # Specifically 2 - Storage (s3, gcs etc), Production RDBMS - Aurora, CloudSQL etc @@ -125,8 +121,6 @@ db: host: postgres dbname: flyte_development - - # -------------------------------------------------------------------- # Specializing your deployment using configuration # ------------------------------------------------------------------- @@ -135,7 +129,6 @@ db: # configmap: - # Task default resources configuration task_resource_defaults: task_resources: @@ -167,9 +160,9 @@ configmap: plugins: k8s: default-env-vars: - - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" - - FLYTE_AWS_ACCESS_KEY_ID: minio - - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage + - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" + - FLYTE_AWS_ACCESS_KEY_ID: minio + - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage default-memory: 200Mi # Logger configuration @@ -204,13 +197,11 @@ configmap: workflow_scheduler: enabled: false - # -- # Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of # various events in their workflows workflow_notifications: - enabled: false - + enabled: false # -- Configuration for the Cluster resource manager component. This is an optional component, that enables automatic # cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain @@ -218,7 +209,6 @@ cluster_resource_manager: # -- Enables the Cluster resource manager component enabled: true - # -------------------------------------------------------- # Optional Plugins # -------------------------------------------------------- @@ -236,12 +226,10 @@ sparkoperator: pytorchoperator: enabled: false - # ---------------------- # Distributed tensorflow training using the TF-Operator # - # ---------------------- # Training on AWS Sagemaker using AWS Sagemaker operator # diff --git a/helm/values.yaml b/helm/values.yaml index 1bcb41a2e4..e831466b2b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -439,7 +439,7 @@ configmap: name: production # -- FlyteAdmin server configuration - server: + adminServer: # Refer to the [server config](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/config#ServerConfig). server: httpPort: 8088 From 1332b9e6c3a9cb41b000fb3c2e273d0acbf6aebe Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 12 May 2021 14:11:56 -0700 Subject: [PATCH 76/92] Fix deployment indentation Signed-off-by: Haytham Abuelfutuh --- Makefile | 10 +++++++++- helm/templates/admin/deployment.yaml | 8 ++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6b38fd26ff..8ddc87a11f 100644 --- a/Makefile +++ b/Makefile @@ -50,4 +50,12 @@ stats: .PHONY: prepare_artifacts prepare_artifacts: - bash script/prepare_artifacts.sh \ No newline at end of file + bash script/prepare_artifacts.sh + +.PHONY: helm_install +helm_install: + helm install flyte --debug ./helm -f helm/values-sandbox.yaml --create-namespace + +.PHONY: helm_upgrade +helm_upgrade: + helm upgrade flyte --debug ./helm -f helm/values-sandbox.yaml --create-namespace diff --git a/helm/templates/admin/deployment.yaml b/helm/templates/admin/deployment.yaml index c6cae03152..8d586b41ee 100644 --- a/helm/templates/admin/deployment.yaml +++ b/helm/templates/admin/deployment.yaml @@ -42,7 +42,7 @@ spec: image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" name: run-migrations - volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} - mountPath: /etc/flyte/config name: config-volume - command: @@ -57,7 +57,7 @@ spec: image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" name: seed-projects - volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} - mountPath: /etc/flyte/config name: config-volume {{- if .Values.cluster_resource_manager.enabled }} @@ -70,7 +70,7 @@ spec: image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" name: sync-cluster-resources - volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} - mountPath: /etc/flyte/clusterresource/templates name: resource-templates - mountPath: /etc/flyte/config @@ -136,7 +136,7 @@ spec: cpu: "0.1" memory: 200Mi serviceAccountName: {{ template "flyteadmin.name" . }} - volumes: {{- include "databaseSecret.volume" . | nindent 6 }} + volumes: {{- include "databaseSecret.volume" . | nindent 8 }} - emptyDir: {} name: shared-data - configMap: From 33626811d11b2da90b9639dbbd818df052ce1fa1 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 12 May 2021 14:25:47 -0700 Subject: [PATCH 77/92] Update image versions Signed-off-by: Haytham Abuelfutuh --- helm/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index e831466b2b..e6e352e584 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -16,7 +16,7 @@ flyteadmin: image: # -- Docker image for Flyteadmin deployment repository: ghcr.io/flyteorg/flyteadmin - tag: v0.4.14 + tag: v0.5.4 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteadmin deployment resources: @@ -61,7 +61,7 @@ datacatalog: image: # -- Docker image for Datacatalog deployment repository: ghcr.io/flyteorg/datacatalog - tag: v0.3.2 + tag: v0.3.4 pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment resources: From 2620dba642768cb931ac10c270b2799a4bfe5a7e Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 19 May 2021 14:18:12 -0700 Subject: [PATCH 78/92] disable tls by default because of: https://github.com/helm/helm/issues/5238 Signed-off-by: Haytham Abuelfutuh --- helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/values.yaml b/helm/values.yaml index e6e352e584..0a03cfe008 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -374,7 +374,7 @@ common: # --- Ingress hostname # host: tls: - enabled: true + enabled: false flyteNamespaceTemplate: # --- Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. enabled: false From de6076b8dd762cea6d040088bbcb3b66d0ce555e Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Thu, 20 May 2021 18:38:38 -0700 Subject: [PATCH 79/92] default imagepullsecrets Signed-off-by: Haytham Abuelfutuh --- helm/values.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/helm/values.yaml b/helm/values.yaml index 0a03cfe008..e0c9d51b62 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -42,6 +42,8 @@ flyteadmin: create: true # -- Annotations for ServiceAccount attached to Flyteadmin pods annotations: {} + # -- ImapgePullSecrets to automatically assign to the service account + imagePullSecrets: {} # -- Annotations for Flyteadmin pods podAnnotations: {} # -- nodeSelector for Flyteadmin deployment @@ -86,6 +88,8 @@ datacatalog: create: true # -- Annotations for ServiceAccount attached to Datacatalog pods annotations: {} + # -- ImapgePullSecrets to automatically assign to the service account + imagePullSecrets: {} # -- Annotations for Datacatalog pods podAnnotations: {} # -- nodeSelector for Datacatalog deployment @@ -127,7 +131,8 @@ flytepropeller: create: true # -- Annotations for ServiceAccount attached to FlytePropeller pods annotations: {} - + # -- ImapgePullSecrets to automatically assign to the service account + imagePullSecrets: {} # -- Annotations for Flytepropeller pods podAnnotations: {} # -- nodeSelector for Flytepropeller deployment @@ -183,6 +188,8 @@ webhook: create: true # -- Annotations for ServiceAccount attached to the webhook annotations: {} + # -- ImapgePullSecrets to automatically assign to the service account + imagePullSecrets: {} # -- Service settings for the webhook service: annotations: From afcfa04bc5bde001fe92c00b3723c9eaa4c14736 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Thu, 20 May 2021 19:13:29 -0700 Subject: [PATCH 80/92] Disable SSL Redirect for GRPC Signed-off-by: Haytham Abuelfutuh --- helm/templates/common/ingress.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/helm/templates/common/ingress.yaml b/helm/templates/common/ingress.yaml index 7b70e4c5d5..10e3f10a82 100644 --- a/helm/templates/common/ingress.yaml +++ b/helm/templates/common/ingress.yaml @@ -191,13 +191,13 @@ spec: rules: - http: paths: - {{- if .Values.common.ingress.albSSLRedirect }} - - backend: - serviceName: ssl-redirect - servicePort: use-annotation - path: /* - pathType: ImplementationSpecific - {{- end }} + # {{- if .Values.common.ingress.albSSLRedirect }} + # - backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + # path: /* + # pathType: ImplementationSpecific + # {{- end }} {{- include "grpcRoutes" . | nindent 10 -}} {{- with .Values.common.ingress.host }} host: {{ . }} From b41f4b6e97dbbeba3a5f49f1012b3f5f62bc1e5a Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Mon, 24 May 2021 16:59:25 -0700 Subject: [PATCH 81/92] Fix spark config map Signed-off-by: Haytham Abuelfutuh --- helm/templates/propeller/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/propeller/configmap.yaml b/helm/templates/propeller/configmap.yaml index 593d968248..3d38de0221 100644 --- a/helm/templates/propeller/configmap.yaml +++ b/helm/templates/propeller/configmap.yaml @@ -36,7 +36,7 @@ data: resource_manager.yaml: | {{ toYaml . | nindent 4 }} {{- end }} {{- if .Values.sparkoperator.enabled }} -{{- with .Values.configmap.spark }} +{{- with .Values.sparkoperator.plugin_config }} spark.yaml: | {{ toYaml . | nindent 4 }} {{- end }} {{- end }} From 51dd9f4fa5418de7201be7685ef94f8327424baa Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Mon, 24 May 2021 17:05:11 -0700 Subject: [PATCH 82/92] Update imagePullSecrets Signed-off-by: Haytham Abuelfutuh --- helm/templates/admin/rbac.yaml | 6 +++--- helm/templates/datacatalog/rbac.yaml | 4 ++-- helm/templates/propeller/rbac.yaml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/helm/templates/admin/rbac.yaml b/helm/templates/admin/rbac.yaml index c14d8ceba3..9bcb0e2ceb 100644 --- a/helm/templates/admin/rbac.yaml +++ b/helm/templates/admin/rbac.yaml @@ -9,10 +9,10 @@ metadata: labels: {{ include "flyteadmin.labels" . | nindent 4 }} {{- with .Values.flyteadmin.serviceAccount.annotations }} annotations: {{ toYaml . | nindent 4 }} - {{- end}} + {{- end }} {{- with .Values.flyteadmin.serviceAccount.imagePullSecrets }} -imagePullSecrets: {{ toYaml . | nindent 4 }} -{{- end}} +imagePullSecrets: {{ toYaml . | nindent 2 }} +{{- end }} {{- end }} --- diff --git a/helm/templates/datacatalog/rbac.yaml b/helm/templates/datacatalog/rbac.yaml index 24f95dc355..86a13b344f 100644 --- a/helm/templates/datacatalog/rbac.yaml +++ b/helm/templates/datacatalog/rbac.yaml @@ -11,6 +11,6 @@ metadata: annotations: {{ toYaml . | nindent 4 }} {{- end }} {{- with .Values.datacatalog.serviceAccount.imagePullSecrets }} -imagePullSecrets: {{ toYaml . | nindent 4 }} -{{- end}} +imagePullSecrets: {{ toYaml . | nindent 2 }} +{{- end }} {{- end }} \ No newline at end of file diff --git a/helm/templates/propeller/rbac.yaml b/helm/templates/propeller/rbac.yaml index b0082fea01..3667b3b8c5 100644 --- a/helm/templates/propeller/rbac.yaml +++ b/helm/templates/propeller/rbac.yaml @@ -11,8 +11,8 @@ metadata: annotations: {{ toYaml . | nindent 4 }} {{- end}} {{- with .Values.flytepropeller.serviceAccount.imagePullSecrets }} -imagePullSecrets: {{ toYaml . | nindent 4 }} -{{- end}} +imagePullSecrets: {{ toYaml . | nindent 2 }} +{{- end }} {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 From 13173e0fd01172d0aceeab515019eab7b8a52704 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Mon, 24 May 2021 17:24:26 -0700 Subject: [PATCH 83/92] Add make helm to generate static helm charts Signed-off-by: Haytham Abuelfutuh --- .github/workflows/tests.yml | 2 + Makefile | 4 + deployment/eks/flyte_helm_generated.yaml | 2077 +++++++++ deployment/gcp/flyte_helm_generated.yaml | 4067 +++++++++++++++++ deployment/sandbox/flyte_helm_generated.yaml | 4200 ++++++++++++++++++ helm/values-gcp.yaml | 4 +- script/generate_helm.sh | 29 + 7 files changed, 10381 insertions(+), 2 deletions(-) create mode 100644 deployment/eks/flyte_helm_generated.yaml create mode 100644 deployment/gcp/flyte_helm_generated.yaml create mode 100644 deployment/sandbox/flyte_helm_generated.yaml create mode 100644 script/generate_helm.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6c690be8be..743b7e1556 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,8 @@ jobs: - uses: actions/checkout@v1 - name: Kustomize and diff run: DELTA_CHECK=true make kustomize + - name: Helm and diff + run: DELTA_CHECK=true make helm - uses: engineerd/setup-kind@v0.5.0 with: version: "v0.10.0" diff --git a/Makefile b/Makefile index 8ddc87a11f..a5449e4b92 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,10 @@ update_boilerplate: kustomize: KUSTOMIZE_VERSION=3.9.2 bash script/generate_kustomize.sh +.PHONY: helm +helm: + bash script/generate_helm.sh + .PHONY: release_automation release_automation: bash script/release.sh diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml new file mode 100644 index 0000000000..bde28c9755 --- /dev/null +++ b/deployment/eks/flyte_helm_generated.yaml @@ -0,0 +1,2077 @@ +--- +# Source: flyte/templates/pytorch-operator/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: pytorch-operator +--- +# Source: flyte/charts/sparkoperator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/charts/sparkoperator/templates/spark-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-spark + namespace: default + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create a Service Account for webhook +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-pod-webhook + namespace: default +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pytorch-operator + namespace: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/admin/deployment.yaml +apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- +# Source: flyte/templates/common/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: db-pass +stringData: + pass.txt: awesomesauce +type: Opaque +--- +# Source: flyte/templates/propeller/deployment.yaml +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: default +type: Opaque +stringData: + client_secret: foobar +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create an empty secret that the first propeller pod will populate +apiVersion: v1 +kind: Secret +metadata: + name: flyte-pod-webhook + namespace: default +type: Opaque +--- +# Source: flyte/templates/admin/cluster_resource_configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: clusterresource-template + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + aa_namespace.yaml: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes + + aab_default_service_account.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + namespace: {{ namespace }} + annotations: + eks.amazonaws.com/role-arn: {{ defaultIamRole }} + + ab_project_resource_quota.yaml: | + apiVersion: v1 + kind: ResourceQuota + metadata: + name: project-quota + namespace: {{ namespace }} + spec: + hard: + limits.cpu: {{ projectQuotaCpu }} + limits.memory: {{ projectQuotaMemory }} + + ac_project_copilot_dataconfig.yaml: | + kind: ConfigMap + apiVersion: v1 + metadata: + name: flyte-data-config + namespace: {{ namespace }} + data: + config.yaml: | + storage: + type: s3 + container: + connection: + auth-type: iam + region: + enable-multicontainer: true + + ad_spark_role.yaml: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: Role + metadata: + name: spark-role + namespace: {{ namespace }} + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - '*' + - apiGroups: + - "" + resources: + - services + verbs: + - '*' + - apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' + + ae_spark_service_account.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: spark + namespace: {{ namespace }} + annotations: + eks.amazonaws.com/role-arn: {{ defaultIamRole }} + + af_spark_role_binding.yaml: | + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: RoleBinding + metadata: + name: spark-role-binding + namespace: {{ namespace }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spark-role + subjects: + - kind: ServiceAccount + name: spark + namespace: {{ namespace }} +--- +# Source: flyte/templates/admin/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-admin-config + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + db.yaml: | + database: + dbname: flyteadmin + host: + passwordPath: /etc/db/pass.txt + port: 5432 + username: flyte + domain.yaml: | + domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + logger.yaml: | + logger: + level: 5 + show-source: true + server.yaml: | + auth: + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + authorizedUris: + - https://localhost:30081 + - http://flyteadmin:80 + - http://flyteadmin.flyte.svc.cluster.local:80 + userAuth: + openId: + baseUrl: https://accounts.google.com + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com + scopes: + - profile + - openid + flyteadmin: + eventVersion: 1 + metadataStoragePrefix: + - metadata + - admin + metricsScope: 'flyte:' + profilerPort: 10254 + roleNameKey: iam.amazonaws.com/role + testing: + host: http://flyteadmin + server: + grpcPort: 8089 + httpPort: 8088 + security: + allowCors: true + allowedHeaders: + - Content-Type + allowedOrigins: + - '*' + secure: false + useAuth: false + remoteData.yaml: | + remoteData: + region: us-east-1 + scheme: local + signedUrls: + durationMinutes: 3 + storage.yaml: | + storage: + type: s3 + container: "" + connection: + auth-type: iam + region: + limits: + maxDownloadMBs: 10 + task_resource_defaults.yaml: | + task_resources: + defaults: + cpu: 1000m + memory: 1000Mi + storage: 1000Mi + limits: + cpu: 2 + gpu: 1 + memory: 8Gi + storage: 2000Mi + cluster_resources.yaml: | + cluster_resources: + customData: + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: 4000Mi + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: 3000Mi + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: 3000Mi + refresh: 5m + templatePath: /etc/flyte/clusterresource/templates +--- +# Source: flyte/templates/console/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-console-config + namespace: default + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + BASE_URL: /console + CONFIG_DIR: /etc/flyte/config + DISABLE_AUTH: "1" +--- +# Source: flyte/templates/datacatalog/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: datacatalog-config + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + db.yaml: | + database: + dbname: flyteadmin + host: + passwordPath: /etc/db/pass.txt + port: 5432 + username: flyte + logger.yaml: | + logger: + level: 5 + show-source: true + server.yaml: | + application: + grpcPort: 8089 + grpcServerReflection: true + httpPort: 8080 + datacatalog: + metrics-scope: datacatalog + profiler-port: 10254 + storage-prefix: metadata/datacatalog + storage.yaml: | + storage: + type: s3 + container: "" + connection: + auth-type: iam + region: + limits: + maxDownloadMBs: 10 +--- +# Source: flyte/templates/propeller/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-propeller-config + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + admin.yaml: | + admin: + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret + endpoint: flyteadmin:81 + insecure: true + event: + capacity: 1000 + rate: 500 + type: admin + catalog.yaml: | + catalog-cache: + endpoint: datacatalog:89 + insecure: true + type: datacatalog + copilot.yaml: | + plugins: + k8s: + co-pilot: + image: ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2 + name: flyte-copilot- + start-timeout: 30s + core.yaml: | + propeller: + downstream-eval-duration: 30s + enable-admin-launcher: true + gc-interval: 12h + kube-client-config: + burst: 25 + qps: 100 + timeout: 30s + leader-election: + enabled: true + lease-duration: 15s + lock-config-map: + name: propeller-leader + namespace: flyte + renew-deadline: 10s + retry-period: 2s + limit-namespace: all + max-workflow-retries: 50 + metadata-prefix: metadata/propeller + metrics-prefix: flyte + prof-port: 10254 + queue: + batch-size: -1 + batching-interval: 2s + queue: + base-delay: 5s + capacity: 1000 + max-delay: 120s + rate: 100 + type: maxof + sub-queue: + capacity: 1000 + rate: 100 + type: bucket + type: batch + rawoutput-prefix: s3://s3-bucket-for-flyte/ + workers: 40 + workflow-reeval-duration: 30s + webhook: + certDir: /etc/webhook/certs + serviceName: flyte-pod-webhook + enabled_plugins.yaml: | + tasks: + task-plugins: + default-for-task-types: + container: container + container_array: k8s-array + hive: athena + pytorch: pytorch + sidecar: sidecar + spark: spark + enabled-plugins: + - container + - sidecar + - spark + - k8s-array + - pytorch + - athena + k8s.yaml: | + plugins: + k8s: + default-cpus: 100m + default-env-vars: [] + default-memory: 100Mi + logger.yaml: | + logger: + level: 5 + show-source: true + resource_manager.yaml: | + propeller: + resourcemanager: + redis: + hostKey: mypassword + hostPath: redis-resource-manager:6379 + resourceMaxQuota: 10000 + type: redis + spark.yaml: | + plugins: + spark: + spark-config-default: + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: BucketOwnerFullControl + - spark.hadoop.fs.s3n.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: org.apache.hadoop.fs.s3a.S3A + - spark.hadoop.fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + - spark.hadoop.fs.AbstractFileSystem.s3.impl: org.apache.hadoop.fs.s3a.S3A + - spark.hadoop.fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: org.apache.hadoop.fs.s3a.S3A + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.blacklist.enabled: "true" + - spark.blacklist.timeout: 5m + - spark.task.maxfailures: "8" + storage.yaml: | + storage: + type: s3 + container: "" + connection: + auth-type: iam + region: + limits: + maxDownloadMBs: 10 + cache.yaml: | + cache: + max_size_mbs: 1024 + target_gc_percent: 70 + task_logs.yaml: | + plugins: + logs: + cloudwatch-enabled: true + cloudwatch-log-group: + cloudwatch-region: + kubernetes-enabled: false +--- +# Source: flyte/templates/propeller/crds/flyteworkflow.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: flyteworkflows.flyte.lyft.com +spec: + group: flyte.lyft.com + names: + kind: FlyteWorkflow + plural: flyteworkflows + shortNames: + - fly + singular: flyteworkflow + scope: Namespaced + version: v1alpha1 +--- +# Source: flyte/templates/pytorch-operator/crds/pytorchjobs.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pytorchjobs.kubeflow.org +spec: + additionalPrinterColumns: + - JSONPath: .status.conditions[-1:].type + name: State + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: kubeflow.org + names: + kind: PyTorchJob + plural: pytorchjobs + singular: pytorchjob + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + pytorchReplicaSpecs: + properties: + Master: + properties: + replicas: + maximum: 1 + minimum: 1 + type: integer + Worker: + properties: + replicas: + minimum: 1 + type: integer + versions: + - name: v1 + served: true + storage: true +--- +# Source: flyte/charts/sparkoperator/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - "*" +- apiGroups: + - "" + resources: + - services + - configmaps + - secrets + verbs: + - create + - get + - delete + - update +- apiGroups: + - extensions + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - get + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - patch +- apiGroups: + - "" + resources: + - resourcequotas + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - update + - delete +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - create + - get + - update + - delete +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + - sparkapplications/status + - scheduledsparkapplications + - scheduledsparkapplications/status + verbs: + - "*" +--- +# Source: flyte/templates/admin/rbac.yaml +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyteadmin + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + - flyte.lyft.com + - rbac.authorization.k8s.io + resources: + - configmaps + - flyteworkflows + - namespaces + - pods + - resourcequotas + - roles + - rolebindings + - secrets + - services + - serviceaccounts + - spark-role + verbs: + - '*' +--- +# Source: flyte/templates/propeller/rbac.yaml +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flytepropeller + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +# Allow RO access to PODS +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +# Allow Event recording access +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - delete + - patch +# Allow Access All plugin objects +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +# Allow Access to CRD +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete + - update +# Allow Access to all resources under flyte.lyft.com +- apiGroups: + - flyte.lyft.com + resources: + - flyteworkflows + - flyteworkflows/finalizers + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - post + - deletecollection +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create a ClusterRole for the webhook +# https://kubernetes.io/docs/admin/authorization/rbac/ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flyte-pod-webhook + namespace: default +rules: + - apiGroups: + - "*" + resources: + - mutatingwebhookconfigurations + - secrets + - pods + verbs: + - get + - create + - update + - patch +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pytorch-operator-admin + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: [] +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pytorchjobs-admin: "true" +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pytorch-operator-edit + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pytorch-operator-view + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - get + - list + - watch +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - services + - endpoints + - events + verbs: + - '*' +--- +# Source: flyte/charts/sparkoperator/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +subjects: + - kind: ServiceAccount + name: flyte-sparkoperator + namespace: default +roleRef: + kind: ClusterRole + name: flyte-sparkoperator + apiGroup: rbac.authorization.k8s.io +--- +# Source: flyte/templates/admin/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flyteadmin-binding + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyteadmin +subjects: +- kind: ServiceAccount + name: flyteadmin + namespace: default +--- +# Source: flyte/templates/propeller/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flytepropeller + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flytepropeller +subjects: +- kind: ServiceAccount + name: flytepropeller + namespace: default +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create a binding from Role -> ServiceAccount +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flyte-pod-webhook + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-pod-webhook +subjects: + - kind: ServiceAccount + name: flyte-pod-webhook + namespace: default +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pytorch-operator +subjects: +- kind: ServiceAccount + name: pytorch-operator + namespace: pytorch-operator +--- +# Source: flyte/charts/sparkoperator/templates/spark-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: spark-role + namespace: default + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - "*" +- apiGroups: + - "" + resources: + - services + verbs: + - "*" +--- +# Source: flyte/charts/sparkoperator/templates/spark-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spark + namespace: default + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +subjects: +- kind: ServiceAccount + name: flyte-spark + namespace: default +roleRef: + kind: Role + name: spark-role + apiGroup: rbac.authorization.k8s.io +--- +# Source: flyte/templates/admin/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyteadmin + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + annotations: + external-dns.alpha.kubernetes.io/hostname: flyte.example.com + projectcontour.io/upstream-protocol.h2c: grpc + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" +spec: + type: LoadBalancer + loadBalancerSourceRanges: + [0.0.0.0/0] + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 81 + protocol: TCP + targetPort: 8089 + - name: redoc + protocol: TCP + port: 87 + targetPort: 8087 + - name: http-metrics + protocol: TCP + port: 10254 + selector: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/console/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyteconsole + namespace: default + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + annotations: + external-dns.alpha.kubernetes.io/hostname: flyte.example.com + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" +spec: + type: LoadBalancer + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/datacatalog/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: datacatalog + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + annotations: + projectcontour.io/upstream-protocol.h2c: grpc + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" +spec: + type: LoadBalancer + ports: + - name: grpc-2 + port: 8089 + protocol: TCP + targetPort: 8089 + - name: http + port: 88 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 89 + protocol: TCP + targetPort: 8089 + selector: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/propeller/webhook.yaml +# Service +apiVersion: v1 +kind: Service +metadata: + name: flyte-pod-webhook + namespace: default + annotations: + projectcontour.io/upstream-protocol.h2c: grpc +spec: + selector: + app: flyte-pod-webhook + ports: + - name: https + protocol: TCP + port: 443 + targetPort: 9443 +--- +# Source: flyte/templates/pytorch-operator/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: pytorch-operator + namespace: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: monitoring-port + port: 8443 + targetPort: 8443 + selector: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/redis/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: redis-resource-manager + namespace: default + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte +--- +# Source: flyte/charts/sparkoperator/templates/deployment.yaml +# If the admission webhook is enabled, then a post-install step is required +# to generate and install the secret in the operator namespace. + +# In the post-install hook, the token corresponding to the operator service account +# is used to authenticate with the Kubernetes API server to install the secret bundle. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + strategy: + type: Recreate + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" + prometheus.io/path: /metrics + labels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + spec: + serviceAccountName: flyte-sparkoperator + securityContext: + {} + containers: + - name: sparkoperator + image: gcr.io/spark-operator/spark-operator:v1beta2-1.2.0-3.0.0 + imagePullPolicy: IfNotPresent + securityContext: + {} + ports: + - name: "metrics" + containerPort: 10254 + + args: + - -v=2 + - -logtostderr + - -namespace= + - -ingress-url-format= + - -controller-threads=10 + - -resync-interval=30 + - -enable-batch-scheduler=false + - -enable-metrics=true + - -metrics-labels=app_type + - -metrics-port=10254 + - -metrics-endpoint=/metrics + - -metrics-prefix= + - -enable-resource-quota-enforcement=false + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 50m + memory: 250Mi +--- +# Source: flyte/templates/admin/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteadmin + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "2a74fd52cd1aea9ab4a05907782a3f50be7777930a292f8c74de653dc71ff19" + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + initContainers: + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - migrate + - run + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: run-migrations + volumeMounts: + - mountPath: /etc/db + name: db-pass + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - migrate + - seed-projects + - flytesnacks + - flytetester + - flyteexamples + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: seed-projects + volumeMounts: + - mountPath: /etc/db + name: db-pass + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - clusterresource + - sync + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: sync-cluster-resources + volumeMounts: + - mountPath: /etc/db + name: db-pass + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + - name: generate-secrets + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + command: ["/bin/sh", "-c"] + args: + [ + "flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + containers: + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - serve + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: flyteadmin + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: 250m + ephemeral-storage: 200Mi + memory: 500Mi + requests: + cpu: 50m + ephemeral-storage: 200Mi + memory: 200Mi + volumeMounts: + - mountPath: /etc/db + name: db-pass + - mountPath: /srv/flyte + name: shared-data + - mountPath: /etc/flyte/config + name: config-volume + - name: auth + mountPath: /etc/secrets/ + - command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: Flyte Admin OpenAPI + - name: SPEC_URL + value: /api/v1/openapi + - name: PORT + value: "8087" + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + name: redoc + ports: + - containerPort: 8087 + resources: + limits: + cpu: "0.1" + memory: 200Mi + serviceAccountName: flyteadmin + volumes: + - name: db-pass + secret: + secretName: db-pass + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-admin-config + name: config-volume + - configMap: + name: clusterresource-template + name: resource-templates + - name: auth + secret: + secretName: flyte-admin-auth + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: flyteadmin + topologyKey: kubernetes.io/hostname +--- +# Source: flyte/templates/console/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteconsole + namespace: default + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "a14c2477f002d1392494a730c37ba7109e61df5cd417d431efa36841ee56bb9" + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "ghcr.io/flyteorg/flyteconsole:v0.20.0" + imagePullPolicy: "IfNotPresent" + name: flyteconsole + envFrom: + - configMapRef: + name: flyte-console-config + ports: + - containerPort: 8080 + resources: + limits: + cpu: 250m + memory: 250Mi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - mountPath: /srv/flyte + name: shared-data + volumes: + - emptyDir: {} + name: shared-data + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: flyteconsole + topologyKey: kubernetes.io/hostname +--- +# Source: flyte/templates/datacatalog/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datacatalog + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "735c43c585eda7d13e8654c91f0cada88063c80535274f5778bc3d7c1bc5545" + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + initContainers: + - command: + - datacatalog + - --config + - /etc/datacatalog/config/*.yaml + - migrate + - run + image: "ghcr.io/flyteorg/datacatalog:v0.3.4" + imagePullPolicy: "IfNotPresent" + name: run-migrations + volumeMounts: + - mountPath: /etc/db + name: db-pass + - mountPath: /etc/datacatalog/config + name: config-volume + containers: + - command: + - datacatalog + - --config + - /etc/datacatalog/config/*.yaml + - serve + image: "ghcr.io/flyteorg/datacatalog:v0.3.4" + imagePullPolicy: "IfNotPresent" + name: datacatalog + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: 500m + ephemeral-storage: 200Mi + memory: 500Mi + requests: + cpu: 50m + ephemeral-storage: 200Mi + memory: 200Mi + volumeMounts: + - mountPath: /etc/db + name: db-pass + - mountPath: /etc/datacatalog/config + name: config-volume + serviceAccountName: datacatalog + volumes: + - name: db-pass + secret: + secretName: db-pass + - emptyDir: {} + name: shared-data + - configMap: + name: datacatalog-config + name: config-volume + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: datacatalog + topologyKey: kubernetes.io/hostname +--- +# Source: flyte/templates/propeller/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: default + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "73d82b14cc9e0ed31b4c5731396ec0a43727ceb7cf3c247c0e3c9023ce1cda2" + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - command: + - flytepropeller + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: "ghcr.io/flyteorg/flytepropeller:v0.10.10" + imagePullPolicy: "IfNotPresent" + name: flytepropeller + ports: + - containerPort: 10254 + resources: + limits: + cpu: 1 + ephemeral-storage: 1Gi + memory: 2Gi + requests: + cpu: 1 + ephemeral-storage: 1Gi + memory: 2Gi + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: auth + mountPath: /etc/secrets/ + serviceAccountName: flytepropeller + volumes: + - configMap: + name: flyte-propeller-config + name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: flytepropeller + topologyKey: kubernetes.io/hostname +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create the actual deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-pod-webhook + namespace: default + labels: + app: flyte-pod-webhook +spec: + selector: + matchLabels: + app: flyte-pod-webhook + template: + metadata: + labels: + app: flyte-pod-webhook + app.kubernetes.io/name: flyte-pod-webhook + app.kubernetes.io/version: v0.10.10 + annotations: + configChecksum: "73d82b14cc9e0ed31b4c5731396ec0a43727ceb7cf3c247c0e3c9023ce1cda2" + spec: + serviceAccountName: flyte-pod-webhook + initContainers: + - name: generate-secrets + image: "ghcr.io/flyteorg/flytepropeller:v0.10.10" + imagePullPolicy: "IfNotPresent" + command: + - flytepropeller + args: + - webhook + - init-certs + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + containers: + - name: webhook + image: "ghcr.io/flyteorg/flytepropeller:v0.10.10" + imagePullPolicy: "IfNotPresent" + command: + - flytepropeller + args: + - webhook + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + readOnly: true + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: config-volume + configMap: + name: flyte-propeller-config + - name: webhook-certs + secret: + secretName: flyte-pod-webhook +--- +# Source: flyte/templates/pytorch-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pytorch-operator + namespace: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + template: + metadata: + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "gcr.io/kubeflow-images-public/pytorch-operator:v1.0.0-g047cf0f" + imagePullPolicy: "IfNotPresent" + name: pytorch-operator + command: + - /pytorch-operator.v1 + - --alsologtostderr + - -v=1 + - --monitoring-port=8443 + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 50m + memory: 250Mi + serviceAccountName: pytorch-operator +--- +# Source: flyte/templates/redis/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: default + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte + serviceName: redis-resource-manager + template: + metadata: + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "docker.io/bitnami/redis:4.0.2-r1" + imagePullPolicy: "IfNotPresent" + name: redis + env: + - name: REDIS_PASSWORD + value: mypassword + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - containerPort: 6379 + name: redis + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 250Mi + volumeMounts: + - mountPath: /bitnami + name: redis-data + + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data +--- +# Source: flyte/templates/admin/cronjob.yaml +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: syncresources + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + schedule: '*/1 * * * *' + jobTemplate: + spec: + template: + spec: + containers: + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - clusterresource + - sync + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: sync-cluster-resources + volumeMounts: + - mountPath: /etc/db + name: db-pass + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: OnFailure + serviceAccountName: flyteadmin + volumes: + - name: db-pass + secret: + secretName: db-pass + - configMap: + name: clusterresource-template + name: resource-templates + - configMap: + name: flyte-admin-config + name: config-volume +--- +# Source: flyte/templates/common/ingress.yaml +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: flyte + namespace: default + annotations: + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2::certificate/e2f04275-2dff-4118-a493-ed3ec8f41605 + alb.ingress.kubernetes.io/group.name: flyte + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/tags: service_instance=production + kubernetes.io/ingress.class: alb +spec: + rules: + - http: + paths: + - path: /* + pathType: ImplementationSpecific + backend: + serviceName: ssl-redirect + servicePort: use-annotation + # This is useful only for frontend development + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 87 + # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml + - path: /console + pathType: ImplementationSpecific + backend: + serviceName: flyteconsole + servicePort: 80 + - path: /console/* + pathType: ImplementationSpecific + backend: + serviceName: flyteconsole + servicePort: 80 + - path: /api + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /api/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /healthcheck + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /v1/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /.well-known + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /.well-known/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /login + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /login/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /logout + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /logout/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /callback + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /callback/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /me + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /config + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /config/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + host: flyte.example.com +# Certain ingress controllers like nginx cannot serve HTTP 1 and GRPC with a single ingress because GRPC can only +# enabled on the ingress object, not on backend services (GRPC annotation is set on the ingress, not on the services). +--- +# Source: flyte/templates/common/ingress.yaml +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: flyte-grpc + namespace: default + annotations: + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2::certificate/e2f04275-2dff-4118-a493-ed3ec8f41605 + alb.ingress.kubernetes.io/group.name: flyte + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/tags: service_instance=production + kubernetes.io/ingress.class: alb + nginx.ingress.kubernetes.io/backend-protocol: GRPC +spec: + rules: + - http: + paths: + # + # - backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + # path: /* + # pathType: ImplementationSpecific + # + # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. + - path: /flyteidl.service.AdminService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AdminService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + host: flyte.example.com diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml new file mode 100644 index 0000000000..3bc0f3ab7c --- /dev/null +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -0,0 +1,4067 @@ +--- +# Source: flyte/templates/pytorch-operator/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: pytorch-operator +--- +# Source: flyte/charts/contour/templates/contour/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-contour-contour + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour +--- +# Source: flyte/charts/contour/templates/envoy/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-contour-envoy + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: envoy +--- +# Source: flyte/charts/sparkoperator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/charts/sparkoperator/templates/spark-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-spark + namespace: default + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/admin/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyteadmin + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/datacatalog/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: datacatalog + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/propeller/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flytepropeller + namespace: default + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create a Service Account for webhook +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-pod-webhook + namespace: default +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pytorch-operator + namespace: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/admin/deployment.yaml +apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- +# Source: flyte/templates/propeller/deployment.yaml +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: default +type: Opaque +stringData: + client_secret: foobar +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create an empty secret that the first propeller pod will populate +apiVersion: v1 +kind: Secret +metadata: + name: flyte-pod-webhook + namespace: default +type: Opaque +--- +# Source: flyte/charts/contour/templates/contour/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-contour + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour +data: + contour.yaml: | + accesslog-format: envoy + disablePermitInsecure: false + envoy-service-name: 'flyte-contour-envoy' + leaderelection: + configmap-namespace: 'default' + tls: {} +--- +# Source: flyte/templates/admin/cluster_resource_configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: clusterresource-template + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + aa_namespace.yaml: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes + + ab_project_resource_quota.yaml: | + apiVersion: v1 + kind: ResourceQuota + metadata: + name: project-quota + namespace: {{ namespace }} + spec: + hard: + limits.cpu: {{ projectQuotaCpu }} + limits.memory: {{ projectQuotaMemory }} + + ac_project_copilot_dataconfig.yaml: | + kind: ConfigMap + apiVersion: v1 + metadata: + name: flyte-data-config + namespace: {{ namespace }} + data: + config.yaml: | + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + type: minio + container: my-s3-bucket + enable-multicontainer: true +--- +# Source: flyte/templates/admin/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-admin-config + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + db.yaml: | + database: {} + domain.yaml: | + domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + logger.yaml: | + logger: + level: 4 + show-source: true + server.yaml: | + auth: + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + authorizedUris: + - https://localhost:30081 + - http://flyteadmin:80 + - http://flyteadmin.flyte.svc.cluster.local:80 + userAuth: + openId: + baseUrl: https://accounts.google.com + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com + scopes: + - profile + - openid + flyteadmin: + eventVersion: 1 + metadataStoragePrefix: + - metadata + - admin + metricsScope: 'flyte:' + profilerPort: 10254 + roleNameKey: iam.amazonaws.com/role + testing: + host: http://flyteadmin + server: + grpcPort: 8089 + httpPort: 8088 + security: + allowCors: true + allowedHeaders: + - Content-Type + - flyte-authorization + allowedOrigins: + - '*' + secure: false + useAuth: false + remoteData.yaml: | + remoteData: + region: us-east-1 + scheme: gcp + signedUrls: + durationMinutes: 3 + storage.yaml: | + storage: + type: stow + stow: + kind: google + config: + json: "" + project_id: + scopes: https://www.googleapis.com/auth/devstorage.read_write + container: "" + limits: + maxDownloadMBs: 10 + task_resource_defaults.yaml: | + task_resources: + defaults: + cpu: 100m + memory: 100Mi + storage: 5Mi + limits: + cpu: 2 + gpu: 1 + memory: 8Gi + storage: 20Mi + cluster_resources.yaml: | + cluster_resources: + customData: + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: 4000Mi + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: 3000Mi + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: 3000Mi + refresh: 5m + templatePath: /etc/flyte/clusterresource/templates +--- +# Source: flyte/templates/console/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-console-config + namespace: default + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + BASE_URL: /console + CONFIG_DIR: /etc/flyte/config + DISABLE_AUTH: "1" +--- +# Source: flyte/templates/datacatalog/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: datacatalog-config + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + db.yaml: | + database: {} + logger.yaml: | + logger: + level: 4 + show-source: true + server.yaml: | + application: + grpcPort: 8089 + grpcServerReflection: true + httpPort: 8080 + datacatalog: + metrics-scope: datacatalog + profiler-port: 10254 + storage-prefix: metadata/datacatalog + storage.yaml: | + storage: + type: stow + stow: + kind: google + config: + json: "" + project_id: + scopes: https://www.googleapis.com/auth/devstorage.read_write + container: "" + limits: + maxDownloadMBs: 10 +--- +# Source: flyte/templates/propeller/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-propeller-config + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + admin.yaml: | + admin: + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret + endpoint: flyteadmin:81 + insecure: true + event: + capacity: 1000 + rate: 500 + type: admin + catalog.yaml: | + catalog-cache: + endpoint: datacatalog:89 + insecure: true + type: datacatalog + copilot.yaml: | + plugins: + k8s: + co-pilot: + image: ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2 + name: flyte-copilot- + start-timeout: 30s + core.yaml: | + propeller: + downstream-eval-duration: 30s + enable-admin-launcher: true + leader-election: + enabled: true + lease-duration: 15s + lock-config-map: + name: propeller-leader + namespace: flyte + renew-deadline: 10s + retry-period: 2s + limit-namespace: all + max-workflow-retries: 30 + metadata-prefix: metadata/propeller + metrics-prefix: flyte + prof-port: 10254 + queue: + batch-size: -1 + batching-interval: 2s + queue: + base-delay: 5s + capacity: 1000 + max-delay: 120s + rate: 100 + type: maxof + sub-queue: + capacity: 100 + rate: 10 + type: bucket + type: batch + rawoutput-prefix: s3://my-s3-bucket/ + workers: 4 + workflow-reeval-duration: 30s + webhook: + certDir: /etc/webhook/certs + serviceName: flyte-pod-webhook + enabled_plugins.yaml: | + tasks: + task-plugins: + default-for-task-types: + container: container + container_array: k8s-array + sidecar: sidecar + enabled-plugins: + - container + - sidecar + - k8s-array + k8s.yaml: | + plugins: + k8s: + default-cpus: 100m + default-env-vars: [] + default-memory: 100Mi + logger.yaml: | + logger: + level: 4 + show-source: true + resource_manager.yaml: | + propeller: + resourcemanager: + redis: + hostKey: mypassword + hostPath: redis-resource-manager:6379 + resourceMaxQuota: 10000 + type: redis + spark.yaml: | + plugins: + spark: + spark-config-default: + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: BucketOwnerFullControl + - spark.hadoop.fs.s3n.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: org.apache.hadoop.fs.s3a.S3A + - spark.hadoop.fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + - spark.hadoop.fs.AbstractFileSystem.s3.impl: org.apache.hadoop.fs.s3a.S3A + - spark.hadoop.fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: org.apache.hadoop.fs.s3a.S3A + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.blacklist.enabled: "true" + - spark.blacklist.timeout: 5m + - spark.task.maxfailures: "8" + storage.yaml: | + storage: + type: stow + stow: + kind: google + config: + json: "" + project_id: + scopes: https://www.googleapis.com/auth/devstorage.read_write + container: "" + limits: + maxDownloadMBs: 10 + cache.yaml: | + cache: + max_size_mbs: 0 + target_gc_percent: 70 + task_logs.yaml: | + plugins: + logs: + cloudwatch-enabled: false + kubernetes-enabled: false +--- +# Source: flyte/charts/contour/templates/00-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.9 + creationTimestamp: null + name: extensionservices.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: ExtensionService + listKind: ExtensionServiceList + plural: extensionservices + shortNames: + - extensionservice + - extensionservices + singular: extensionservice + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ExtensionService is the schema for the Contour extension services API. An ExtensionService resource binds a network service to the Contour API so that Contour API features can be implemented by collaborating components. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. + properties: + loadBalancerPolicy: + description: The policy for load balancing GRPC service requests. Note that the `Cookie` load balancing strategy cannot be used here. + properties: + strategy: + description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. + type: string + type: object + protocol: + description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. + enum: + - h2 + - h2c + type: string + protocolVersion: + description: This field sets the version of the GRPC protocol that Envoy uses to send requests to the extension service. Since Contour always uses the v3 Envoy API, this is currently fixed at "v3". However, other protocol options will be available in future. + enum: + - v3 + type: string + services: + description: Services specifies the set of Kubernetes Service resources that receive GRPC extension API requests. If no weights are specified for any of the entries in this array, traffic will be spread evenly across all the services. Otherwise, traffic is balanced proportionally to the Weight field in each entry. + items: + description: ExtensionServiceTarget defines an Kubernetes Service to target with extension service traffic. + properties: + name: + description: Name is the name of Kubernetes service that will accept service traffic. + type: string + port: + description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. + exclusiveMaximum: true + maximum: 65536 + minimum: 1 + type: integer + weight: + description: Weight defines proportion of traffic to balance to the Kubernetes Service. + format: int32 + type: integer + required: + - name + - port + type: object + minItems: 1 + type: array + timeoutPolicy: + description: The timeout policy for requests to the services. + properties: + idle: + description: Timeout after which, if there are no active requests for this route, the connection between Envoy and the backend or Envoy and the external client will be closed. If not specified, there is no per-route idle timeout, though a connection manager-wide stream_idle_timeout default of 5m still applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + response: + description: Timeout for receiving a response from the server after processing a request from client. If not supplied, Envoy's default value of 15s applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + type: object + validation: + description: UpstreamValidation defines how to verify the backend service's certificate + properties: + caSecret: + description: Name of the Kubernetes secret be used to validate the certificate presented by the backend + type: string + subjectName: + description: Key which is expected to be present in the 'subjectAltName' of the presented certificate + type: string + required: + - caSecret + - subjectName + type: object + required: + - services + type: object + status: + description: ExtensionServiceStatus defines the observed state of an ExtensionService resource. + properties: + conditions: + description: "Conditions contains the current status of the ExtensionService resource. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. \n Contour will not modify any other Conditions set in this block, in case some other controller wants to add a Condition." + items: + description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." + properties: + errors: + description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: flyte/charts/contour/templates/00-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.9 + creationTimestamp: null + name: httpproxies.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: HTTPProxy + listKind: HTTPProxyList + plural: httpproxies + shortNames: + - proxy + - proxies + singular: httpproxy + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Fully qualified domain name + jsonPath: .spec.virtualhost.fqdn + name: FQDN + type: string + - description: Secret with TLS credentials + jsonPath: .spec.virtualhost.tls.secretName + name: TLS Secret + type: string + - description: The current status of the HTTPProxy + jsonPath: .status.currentStatus + name: Status + type: string + - description: Description of the current status + jsonPath: .status.description + name: Status Description + type: string + name: v1 + schema: + openAPIV3Schema: + description: HTTPProxy is an Ingress CRD specification. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HTTPProxySpec defines the spec of the CRD. + properties: + ggvar: + description: Testing new param for CRD + type: string + includes: + description: Includes allow for specific routing configuration to be included from another HTTPProxy, possibly in another namespace. + items: + description: Include describes a set of policies that can be applied to an HTTPProxy in a namespace. + properties: + conditions: + description: 'Conditions are a set of rules that are applied to included HTTPProxies. In effect, they are added onto the Conditions of included HTTPProxy Route structs. When applied, they are merged using AND, with one exception: There can be only one Prefix MatchCondition per Conditions slice. More than one Prefix, or contradictory Conditions, will make the include invalid.' + items: + description: MatchCondition are a general holder for matching rules for HTTPProxies. One of Prefix or Header must be provided. + properties: + header: + description: Header specifies the header condition to match. + properties: + contains: + description: Contains specifies a substring that must be present in the header value. + type: string + exact: + description: Exact specifies a string that the header value must be equal to. + type: string + name: + description: Name is the name of the header to match against. Name is required. Header names are case insensitive. + type: string + notcontains: + description: NotContains specifies a substring that must not be present in the header value. + type: string + notexact: + description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value. + type: string + present: + description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent. + type: boolean + required: + - name + type: object + prefix: + description: Prefix defines a prefix match for a request. + type: string + type: object + type: array + name: + description: Name of the HTTPProxy + type: string + namespace: + description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. + type: string + required: + - name + type: object + type: array + routes: + description: Routes are the ingress routes. If TCPProxy is present, Routes is ignored. + items: + description: Route contains the set of routes for a virtual host. + properties: + authPolicy: + description: AuthPolicy updates the authorization policy that was set on the root HTTPProxy object for client requests that match this route. + properties: + context: + additionalProperties: + type: string + description: Context is a set of key/value pairs that are sent to the authentication server in the check request. If a context is provided at an enclosing scope, the entries are merged such that the inner scope overrides matching keys from the outer scope. + type: object + disabled: + description: When true, this field disables client request authentication for the scope of the policy. + type: boolean + type: object + conditions: + description: 'Conditions are a set of rules that are applied to a Route. When applied, they are merged using AND, with one exception: There can be only one Prefix MatchCondition per Conditions slice. More than one Prefix, or contradictory Conditions, will make the route invalid.' + items: + description: MatchCondition are a general holder for matching rules for HTTPProxies. One of Prefix or Header must be provided. + properties: + header: + description: Header specifies the header condition to match. + properties: + contains: + description: Contains specifies a substring that must be present in the header value. + type: string + exact: + description: Exact specifies a string that the header value must be equal to. + type: string + name: + description: Name is the name of the header to match against. Name is required. Header names are case insensitive. + type: string + notcontains: + description: NotContains specifies a substring that must not be present in the header value. + type: string + notexact: + description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value. + type: string + present: + description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent. + type: boolean + required: + - name + type: object + prefix: + description: Prefix defines a prefix match for a request. + type: string + type: object + type: array + enableWebsockets: + description: Enables websocket support for the route. + type: boolean + healthCheckPolicy: + description: The health check policy for this route. + properties: + healthyThresholdCount: + description: The number of healthy health checks required before a host is marked healthy + format: int64 + minimum: 0 + type: integer + host: + description: The value of the host header in the HTTP health check request. If left empty (default value), the name "contour-envoy-healthcheck" will be used. + type: string + intervalSeconds: + description: The interval (seconds) between health checks + format: int64 + type: integer + path: + description: HTTP endpoint used to perform health checks on upstream service + type: string + timeoutSeconds: + description: The time to wait (seconds) for a health check response + format: int64 + type: integer + unhealthyThresholdCount: + description: The number of unhealthy health checks required before a host is marked unhealthy + format: int64 + minimum: 0 + type: integer + required: + - path + type: object + loadBalancerPolicy: + description: The load balancing policy for this route. + properties: + strategy: + description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. + type: string + type: object + pathRewritePolicy: + description: The policy for rewriting the path of the request URL after the request has been routed to a Service. + properties: + replacePrefix: + description: ReplacePrefix describes how the path prefix should be replaced. + items: + description: ReplacePrefix describes a path prefix replacement. + properties: + prefix: + description: "Prefix specifies the URL path prefix to be replaced. \n If Prefix is specified, it must exactly match the MatchCondition prefix that is rendered by the chain of including HTTPProxies and only that path prefix will be replaced by Replacement. This allows HTTPProxies that are included through multiple roots to only replace specific path prefixes, leaving others unmodified. \n If Prefix is not specified, all routing prefixes rendered by the include chain will be replaced." + minLength: 1 + type: string + replacement: + description: Replacement is the string that the routing path prefix will be replaced with. This must not be empty. + minLength: 1 + type: string + required: + - replacement + type: object + type: array + type: object + permitInsecure: + description: Allow this path to respond to insecure requests over HTTP which are normally not permitted when a `virtualhost.tls` block is present. + type: boolean + requestHeadersPolicy: + description: The policy for managing request headers during proxying. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + responseHeadersPolicy: + description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + retryPolicy: + description: The retry policy for this route. + properties: + count: + description: NumRetries is maximum allowed number of retries. If not supplied, the number of retries is one. + format: int64 + minimum: 0 + type: integer + perTryTimeout: + description: PerTryTimeout specifies the timeout per retry attempt. Ignored if NumRetries is not supplied. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + retriableStatusCodes: + description: "RetriableStatusCodes specifies the HTTP status codes that should be retried. \n This field is only respected when you include `retriable-status-codes` in the `RetryOn` field." + items: + format: int32 + type: integer + type: array + retryOn: + description: "RetryOn specifies the conditions on which to retry a request. \n Supported [HTTP conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-on): \n - `5xx` - `gateway-error` - `reset` - `connect-failure` - `retriable-4xx` - `refused-stream` - `retriable-status-codes` - `retriable-headers` \n Supported [gRPC conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-grpc-on): \n - `cancelled` - `deadline-exceeded` - `internal` - `resource-exhausted` - `unavailable`" + items: + description: RetryOn is a string type alias with validation to ensure that the value is valid. + enum: + - 5xx + - gateway-error + - reset + - connect-failure + - retriable-4xx + - refused-stream + - retriable-status-codes + - retriable-headers + - cancelled + - deadline-exceeded + - internal + - resource-exhausted + - unavailable + type: string + type: array + type: object + services: + description: Services are the services to proxy traffic. + items: + description: Service defines an Kubernetes Service to proxy traffic. + properties: + mirror: + description: If Mirror is true the Service will receive a read only mirror of the traffic for this route. + type: boolean + name: + description: Name is the name of Kubernetes service to proxy traffic. Names defined here will be used to look up corresponding endpoints which contain the ips to route. + type: string + port: + description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. + exclusiveMaximum: true + maximum: 65536 + minimum: 1 + type: integer + protocol: + description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. + enum: + - h2 + - h2c + - tls + type: string + requestHeadersPolicy: + description: The policy for managing request headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + responseHeadersPolicy: + description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + validation: + description: UpstreamValidation defines how to verify the backend service's certificate + properties: + caSecret: + description: Name of the Kubernetes secret be used to validate the certificate presented by the backend + type: string + subjectName: + description: Key which is expected to be present in the 'subjectAltName' of the presented certificate + type: string + required: + - caSecret + - subjectName + type: object + weight: + description: Weight defines percentage of traffic to balance traffic + format: int64 + minimum: 0 + type: integer + required: + - name + - port + type: object + minItems: 1 + type: array + timeoutPolicy: + description: The timeout policy for this route. + properties: + idle: + description: Timeout after which, if there are no active requests for this route, the connection between Envoy and the backend or Envoy and the external client will be closed. If not specified, there is no per-route idle timeout, though a connection manager-wide stream_idle_timeout default of 5m still applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + response: + description: Timeout for receiving a response from the server after processing a request from client. If not supplied, Envoy's default value of 15s applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + type: object + required: + - services + type: object + type: array + tcpproxy: + description: TCPProxy holds TCP proxy information. + properties: + healthCheckPolicy: + description: The health check policy for this tcp proxy + properties: + healthyThresholdCount: + description: The number of healthy health checks required before a host is marked healthy + format: int32 + type: integer + intervalSeconds: + description: The interval (seconds) between health checks + format: int64 + type: integer + timeoutSeconds: + description: The time to wait (seconds) for a health check response + format: int64 + type: integer + unhealthyThresholdCount: + description: The number of unhealthy health checks required before a host is marked unhealthy + format: int32 + type: integer + type: object + include: + description: Include specifies that this tcpproxy should be delegated to another HTTPProxy. + properties: + name: + description: Name of the child HTTPProxy + type: string + namespace: + description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. + type: string + required: + - name + type: object + includes: + description: "IncludesDeprecated allow for specific routing configuration to be appended to another HTTPProxy in another namespace. \n Exists due to a mistake when developing HTTPProxy and the field was marked plural when it should have been singular. This field should stay to not break backwards compatibility to v1 users." + properties: + name: + description: Name of the child HTTPProxy + type: string + namespace: + description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. + type: string + required: + - name + type: object + loadBalancerPolicy: + description: The load balancing policy for the backend services. + properties: + strategy: + description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. + type: string + type: object + services: + description: Services are the services to proxy traffic + items: + description: Service defines an Kubernetes Service to proxy traffic. + properties: + mirror: + description: If Mirror is true the Service will receive a read only mirror of the traffic for this route. + type: boolean + name: + description: Name is the name of Kubernetes service to proxy traffic. Names defined here will be used to look up corresponding endpoints which contain the ips to route. + type: string + port: + description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. + exclusiveMaximum: true + maximum: 65536 + minimum: 1 + type: integer + protocol: + description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. + enum: + - h2 + - h2c + - tls + type: string + requestHeadersPolicy: + description: The policy for managing request headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + responseHeadersPolicy: + description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + validation: + description: UpstreamValidation defines how to verify the backend service's certificate + properties: + caSecret: + description: Name of the Kubernetes secret be used to validate the certificate presented by the backend + type: string + subjectName: + description: Key which is expected to be present in the 'subjectAltName' of the presented certificate + type: string + required: + - caSecret + - subjectName + type: object + weight: + description: Weight defines percentage of traffic to balance traffic + format: int64 + minimum: 0 + type: integer + required: + - name + - port + type: object + type: array + type: object + virtualhost: + description: Virtualhost appears at most once. If it is present, the object is considered to be a "root" HTTPProxy. + properties: + authorization: + description: This field configures an extension service to perform authorization for this virtual host. Authorization can only be configured on virtual hosts that have TLS enabled. If the TLS configuration requires client certificate /validation, the client certificate is always included in the authentication check request. + properties: + authPolicy: + description: AuthPolicy sets a default authorization policy for client requests. This policy will be used unless overridden by individual routes. + properties: + context: + additionalProperties: + type: string + description: Context is a set of key/value pairs that are sent to the authentication server in the check request. If a context is provided at an enclosing scope, the entries are merged such that the inner scope overrides matching keys from the outer scope. + type: object + disabled: + description: When true, this field disables client request authentication for the scope of the policy. + type: boolean + type: object + extensionRef: + description: ExtensionServiceRef specifies the extension resource that will authorize client requests. + properties: + apiVersion: + description: API version of the referent. If this field is not specified, the default "projectcontour.io/v1alpha1" will be used + minLength: 1 + type: string + name: + description: "Name of the referent. \n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" + minLength: 1 + type: string + namespace: + description: "Namespace of the referent. If this field is not specifies, the namespace of the resource that targets the referent will be used. \n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/" + minLength: 1 + type: string + type: object + failOpen: + description: If FailOpen is true, the client request is forwarded to the upstream service even if the authorization server fails to respond. This field should not be set in most cases. It is intended for use only while migrating applications from internal authorization to Contour external authorization. + type: boolean + responseTimeout: + description: ResponseTimeout configures maximum time to wait for a check response from the authorization server. Timeout durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". The string "infinity" is also a valid input and specifies no timeout. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + required: + - extensionRef + type: object + corsPolicy: + description: Specifies the cross-origin policy to apply to the VirtualHost. + properties: + allowCredentials: + description: Specifies whether the resource allows credentials. + type: boolean + allowHeaders: + description: AllowHeaders specifies the content for the *access-control-allow-headers* header. + items: + description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. + pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ + type: string + type: array + allowMethods: + description: AllowMethods specifies the content for the *access-control-allow-methods* header. + items: + description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. + pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ + type: string + type: array + allowOrigin: + description: AllowOrigin specifies the origins that will be allowed to do CORS requests. "*" means allow any origin. + items: + type: string + type: array + exposeHeaders: + description: ExposeHeaders Specifies the content for the *access-control-expose-headers* header. + items: + description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. + pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ + type: string + type: array + maxAge: + description: MaxAge indicates for how long the results of a preflight request can be cached. MaxAge durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Only positive values are allowed while 0 disables the cache requiring a preflight OPTIONS check for all cross-origin requests. + type: string + required: + - allowMethods + - allowOrigin + type: object + fqdn: + description: The fully qualified domain name of the root of the ingress tree all leaves of the DAG rooted at this object relate to the fqdn. + type: string + tls: + description: If present the fields describes TLS properties of the virtual host. The SNI names that will be matched on are described in fqdn, the tls.secretName secret must contain a certificate that itself contains a name that matches the FQDN. + properties: + clientValidation: + description: "ClientValidation defines how to verify the client certificate when an external client establishes a TLS connection to Envoy. \n This setting: \n 1. Enables TLS client certificate validation. 2. Requires clients to present a TLS certificate (i.e. not optional validation). 3. Specifies how the client certificate will be validated." + properties: + caSecret: + description: Name of a Kubernetes secret that contains a CA certificate bundle. The client certificate must validate against the certificates in the bundle. + minLength: 1 + type: string + required: + - caSecret + type: object + enableFallbackCertificate: + description: EnableFallbackCertificate defines if the vhost should allow a default certificate to be applied which handles all requests which don't match the SNI defined in this vhost. + type: boolean + minimumProtocolVersion: + description: Minimum TLS version this vhost should negotiate + type: string + passthrough: + description: Passthrough defines whether the encrypted TLS handshake will be passed through to the backing cluster. Either Passthrough or SecretName must be specified, but not both. + type: boolean + secretName: + description: SecretName is the name of a TLS secret in the current namespace. Either SecretName or Passthrough must be specified, but not both. If specified, the named secret must contain a matching certificate for the virtual host's FQDN. + type: string + type: object + required: + - fqdn + type: object + type: object + status: + description: Status is a container for computed information about the HTTPProxy. + properties: + conditions: + description: "Conditions contains information about the current status of the HTTPProxy, in an upstream-friendly container. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, and vice versa. \n Contour will leave untouched any other Conditions set in this block, in case some other controller wants to add a Condition. \n If you are another controller owner and wish to add a condition, you *should* namespace your condition with a label, like `controller.domain.com/ConditionName`." + items: + description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." + properties: + errors: + description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + currentStatus: + type: string + description: + type: string + loadBalancer: + description: LoadBalancer contains the current status of the load balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) + type: string + type: object + type: array + type: object + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: flyte/charts/contour/templates/00-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.9 + creationTimestamp: null + name: tlscertificatedelegations.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: TLSCertificateDelegation + listKind: TLSCertificateDelegationList + plural: tlscertificatedelegations + shortNames: + - tlscerts + singular: tlscertificatedelegation + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: TLSCertificateDelegation is an TLS Certificate Delegation CRD specification. See design/tls-certificate-delegation.md for details. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TLSCertificateDelegationSpec defines the spec of the CRD + properties: + delegations: + items: + description: CertificateDelegation maps the authority to reference a secret in the current namespace to a set of namespaces. + properties: + secretName: + description: required, the name of a secret in the current namespace. + type: string + targetNamespaces: + description: required, the namespaces the authority to reference the the secret will be delegated to. If TargetNamespaces is nil or empty, the CertificateDelegation' is ignored. If the TargetNamespace list contains the character, "*" the secret will be delegated to all namespaces. + items: + type: string + type: array + required: + - secretName + - targetNamespaces + type: object + type: array + required: + - delegations + type: object + status: + description: TLSCertificateDelegationStatus allows for the status of the delegation to be presented to the user. + properties: + conditions: + description: "Conditions contains information about the current status of the HTTPProxy, in an upstream-friendly container. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, and vice versa. \n Contour will leave untouched any other Conditions set in this block, in case some other controller wants to add a Condition. \n If you are another controller owner and wish to add a condition, you *should* namespace your condition with a label, like `controller.domain.com\\ConditionName`." + items: + description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." + properties: + errors: + description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: flyte/templates/propeller/crds/flyteworkflow.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: flyteworkflows.flyte.lyft.com +spec: + group: flyte.lyft.com + names: + kind: FlyteWorkflow + plural: flyteworkflows + shortNames: + - fly + singular: flyteworkflow + scope: Namespaced + version: v1alpha1 +--- +# Source: flyte/templates/pytorch-operator/crds/pytorchjobs.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pytorchjobs.kubeflow.org +spec: + additionalPrinterColumns: + - JSONPath: .status.conditions[-1:].type + name: State + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: kubeflow.org + names: + kind: PyTorchJob + plural: pytorchjobs + singular: pytorchjob + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + pytorchReplicaSpecs: + properties: + Master: + properties: + replicas: + maximum: 1 + minimum: 1 + type: integer + Worker: + properties: + replicas: + minimum: 1 + type: integer + versions: + - name: v1 + served: true + storage: true +--- +# Source: flyte/charts/contour/templates/contour/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyte-contour-contour + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - update + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - apiGroups: + - networking.k8s.io + resources: + - gatewayclasses + - gateways + - httproutes + - tcproutes + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - create + - get + - update + - apiGroups: + - projectcontour.io + resources: + - extensionservices + verbs: + - get + - list + - watch + - apiGroups: + - projectcontour.io + resources: + - extensionservices/status + verbs: + - create + - get + - update + - apiGroups: + - projectcontour.io + resources: + - httpproxies + - tlscertificatedelegations + verbs: + - get + - list + - watch + - apiGroups: + - projectcontour.io + resources: + - httpproxies/status + verbs: + - create + - get + - update +--- +# Source: flyte/charts/sparkoperator/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - "*" +- apiGroups: + - "" + resources: + - services + - configmaps + - secrets + verbs: + - create + - get + - delete + - update +- apiGroups: + - extensions + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - get + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - patch +- apiGroups: + - "" + resources: + - resourcequotas + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - update + - delete +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - create + - get + - update + - delete +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + - sparkapplications/status + - scheduledsparkapplications + - scheduledsparkapplications/status + verbs: + - "*" +--- +# Source: flyte/templates/admin/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyteadmin + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + - flyte.lyft.com + - rbac.authorization.k8s.io + resources: + - configmaps + - flyteworkflows + - namespaces + - pods + - resourcequotas + - roles + - rolebindings + - secrets + - services + - serviceaccounts + - spark-role + verbs: + - '*' +--- +# Source: flyte/templates/propeller/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flytepropeller + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +# Allow RO access to PODS +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +# Allow Event recording access +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - delete + - patch +# Allow Access All plugin objects +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +# Allow Access to CRD +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete + - update +# Allow Access to all resources under flyte.lyft.com +- apiGroups: + - flyte.lyft.com + resources: + - flyteworkflows + - flyteworkflows/finalizers + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - post + - deletecollection +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create a ClusterRole for the webhook +# https://kubernetes.io/docs/admin/authorization/rbac/ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flyte-pod-webhook + namespace: default +rules: + - apiGroups: + - "*" + resources: + - mutatingwebhookconfigurations + - secrets + - pods + verbs: + - get + - create + - update + - patch +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pytorch-operator-admin + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: [] +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pytorchjobs-admin: "true" +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pytorch-operator-edit + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pytorch-operator-view + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - get + - list + - watch +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - services + - endpoints + - events + verbs: + - '*' +--- +# Source: flyte/charts/contour/templates/contour/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flyte-contour-contour + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-contour-contour +subjects: + - kind: ServiceAccount + name: flyte-contour-contour + namespace: default +--- +# Source: flyte/charts/sparkoperator/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +subjects: + - kind: ServiceAccount + name: flyte-sparkoperator + namespace: default +roleRef: + kind: ClusterRole + name: flyte-sparkoperator + apiGroup: rbac.authorization.k8s.io +--- +# Source: flyte/templates/admin/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flyteadmin-binding + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyteadmin +subjects: +- kind: ServiceAccount + name: flyteadmin + namespace: default +--- +# Source: flyte/templates/propeller/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flytepropeller + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flytepropeller +subjects: +- kind: ServiceAccount + name: flytepropeller + namespace: default +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create a binding from Role -> ServiceAccount +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flyte-pod-webhook + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-pod-webhook +subjects: + - kind: ServiceAccount + name: flyte-pod-webhook + namespace: default +--- +# Source: flyte/templates/pytorch-operator/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pytorch-operator +subjects: +- kind: ServiceAccount + name: pytorch-operator + namespace: pytorch-operator +--- +# Source: flyte/charts/sparkoperator/templates/spark-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: spark-role + namespace: default + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - "*" +- apiGroups: + - "" + resources: + - services + verbs: + - "*" +--- +# Source: flyte/charts/sparkoperator/templates/spark-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spark + namespace: default + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +subjects: +- kind: ServiceAccount + name: flyte-spark + namespace: default +roleRef: + kind: Role + name: spark-role + apiGroup: rbac.authorization.k8s.io +--- +# Source: flyte/charts/contour/templates/contour/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyte-contour + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour +spec: + ports: + - port: 8001 + name: xds + protocol: TCP + targetPort: 8001 + selector: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: contour + type: ClusterIP +--- +# Source: flyte/charts/contour/templates/envoy/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyte-contour-envoy + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: envoy + annotations: + # This annotation puts the AWS ELB into "TCP" mode so that it does not + # do HTTP negotiation for HTTPS connections at the ELB edge. + # The downside of this is the remote IP address of all connections will + # appear to be the internal address of the ELB. See docs/proxy-proto.md + # for information about enabling the PROXY protocol on the ELB to recover + # the original remote IP address. + # We don't set this for nlb, per the contour docs. + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp +spec: + externalTrafficPolicy: "Local" + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: envoy + type: LoadBalancer +--- +# Source: flyte/templates/admin/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyteadmin + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + annotations: + projectcontour.io/upstream-protocol.h2c: grpc +spec: + type: ClusterIP + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 81 + protocol: TCP + targetPort: 8089 + - name: redoc + protocol: TCP + port: 87 + targetPort: 8087 + - name: http-metrics + protocol: TCP + port: 10254 + selector: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/console/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyteconsole + namespace: default + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/datacatalog/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: datacatalog + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + annotations: + projectcontour.io/upstream-protocol.h2c: grpc +spec: + type: NodePort + ports: + - name: grpc-2 + port: 8089 + protocol: TCP + targetPort: 8089 + - name: http + port: 88 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 89 + protocol: TCP + targetPort: 8089 + selector: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/minio/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: default + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: minio + port: 9000 + protocol: TCP + targetPort: minio + selector: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/postgres/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: default + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: postgres + port: 5432 + protocol: TCP + targetPort: postgres + selector: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/propeller/webhook.yaml +# Service +apiVersion: v1 +kind: Service +metadata: + name: flyte-pod-webhook + namespace: default + annotations: + projectcontour.io/upstream-protocol.h2c: grpc +spec: + selector: + app: flyte-pod-webhook + ports: + - name: https + protocol: TCP + port: 443 + targetPort: 9443 +--- +# Source: flyte/templates/pytorch-operator/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: pytorch-operator + namespace: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: monitoring-port + port: 8443 + targetPort: 8443 + selector: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/redis/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: redis-resource-manager + namespace: default + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte +--- +# Source: flyte/charts/contour/templates/envoy/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: flyte-contour-envoy + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: envoy +spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: envoy + template: + metadata: + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: envoy + spec: + + affinity: + podAffinity: + + podAntiAffinity: + + nodeAffinity: + + terminationGracePeriodSeconds: 300 + hostNetwork: false + dnsPolicy: ClusterFirst + containers: + - command: + - contour + args: + - envoy + - shutdown-manager + image: docker.io/bitnami/contour:1.12.0-debian-10-r0 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - contour + - envoy + - shutdown + livenessProbe: + httpGet: + path: /healthz + port: 8090 + initialDelaySeconds: 120 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + name: shutdown-manager + - command: + - envoy + args: + - -c + - /config/envoy.json + - --service-cluster $(CONTOUR_NAMESPACE) + - --service-node $(ENVOY_POD_NAME) + - --log-level info + image: docker.io/bitnami/envoy:1.16.2-debian-10-r54 + imagePullPolicy: IfNotPresent + name: envoy + securityContext: + runAsUser: 0 + env: + - name: CONTOUR_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + ports: + - containerPort: 80 + hostPort: 80 + name: http + protocol: TCP + - containerPort: 443 + hostPort: 443 + name: https + protocol: TCP + - containerPort: 8002 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: 8002 + initialDelaySeconds: 10 + periodSeconds: 3 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /ready + port: 8002 + initialDelaySeconds: 120 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - name: envoy-config + mountPath: /config + - name: envoycert + mountPath: /certs + lifecycle: + preStop: + httpGet: + path: /shutdown + port: 8090 + scheme: HTTP + initContainers: + - command: + - contour + args: + - bootstrap + - /config/envoy.json + - --xds-address=flyte-contour + - --xds-port=8001 + - --resources-dir=/config/resources + - --envoy-cafile=/certs/ca.crt + - --envoy-cert-file=/certs/tls.crt + - --envoy-key-file=/certs/tls.key + image: docker.io/bitnami/contour:1.12.0-debian-10-r0 + imagePullPolicy: IfNotPresent + name: envoy-initconfig + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - name: envoy-config + mountPath: /config + - name: envoycert + mountPath: /certs + readOnly: true + env: + - name: CONTOUR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + automountServiceAccountToken: false + serviceAccountName: flyte-contour-envoy + volumes: + - name: envoy-config + emptyDir: {} + - name: envoycert + secret: + secretName: envoycert + restartPolicy: Always +--- +# Source: flyte/charts/contour/templates/contour/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-contour-contour + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # This value of maxSurge means that during a rolling update + # the new ReplicaSet will be created first. + maxSurge: 50% + selector: + matchLabels: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: contour + template: + metadata: + annotations: + checksum/config: 1c6f180e2479c6aff9159e6eea066d5238a8c9ee1f38f19a6de2f070a88e5ad9 + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour + spec: + + affinity: + podAffinity: + + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: contour + namespaces: + - "default" + topologyKey: kubernetes.io/hostname + weight: 1 + nodeAffinity: + + containers: + - command: + - contour + args: + - serve + - --incluster + - --xds-address=0.0.0.0 + - --xds-port=8001 + - --envoy-service-http-port=80 + - --envoy-service-https-port=443 + - --contour-cafile=/certs/ca.crt + - --contour-cert-file=/certs/tls.crt + - --contour-key-file=/certs/tls.key + - --config-path=/config/contour.yaml + image: docker.io/bitnami/contour:1.12.0-debian-10-r0 + imagePullPolicy: IfNotPresent + name: contour + ports: + - containerPort: 8001 + name: xds + protocol: TCP + - containerPort: 8000 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8000 + initialDelaySeconds: 120 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + readinessProbe: + tcpSocket: + port: 8001 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - name: contourcert + mountPath: /certs + readOnly: true + - name: contour-config + mountPath: /config + readOnly: true + env: + - name: CONTOUR_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + dnsPolicy: ClusterFirst + serviceAccountName: flyte-contour-contour + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: + runAsNonRoot: true + volumes: + - name: contourcert + secret: + secretName: contourcert + - name: contour-config + configMap: + name: flyte-contour + defaultMode: 0644 + items: + - key: contour.yaml + path: contour.yaml +--- +# Source: flyte/charts/sparkoperator/templates/deployment.yaml +# If the admission webhook is enabled, then a post-install step is required +# to generate and install the secret in the operator namespace. + +# In the post-install hook, the token corresponding to the operator service account +# is used to authenticate with the Kubernetes API server to install the secret bundle. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + strategy: + type: Recreate + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" + prometheus.io/path: /metrics + labels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + spec: + serviceAccountName: flyte-sparkoperator + securityContext: + {} + containers: + - name: sparkoperator + image: gcr.io/spark-operator/spark-operator:v1beta2-1.2.0-3.0.0 + imagePullPolicy: IfNotPresent + securityContext: + {} + ports: + - name: "metrics" + containerPort: 10254 + + args: + - -v=2 + - -logtostderr + - -namespace= + - -ingress-url-format= + - -controller-threads=10 + - -resync-interval=30 + - -enable-batch-scheduler=false + - -enable-metrics=true + - -metrics-labels=app_type + - -metrics-port=10254 + - -metrics-endpoint=/metrics + - -metrics-prefix= + - -enable-resource-quota-enforcement=false + resources: + limits: + cpu: 1000m + memory: 500M + requests: + cpu: 10m + memory: 50M +--- +# Source: flyte/templates/admin/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteadmin + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "8b55c48b5efdac613e43466b270be585430a0585ad2865c1ac6c6358fa5643e" + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + initContainers: + - name: check-db-ready + image: postgres:10.16-alpine + command: + - sh + - -c + - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done; + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - migrate + - run + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: run-migrations + volumeMounts: + + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - migrate + - seed-projects + - flytesnacks + - flytetester + - flyteexamples + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: seed-projects + volumeMounts: + + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - clusterresource + - sync + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: sync-cluster-resources + volumeMounts: + + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + - name: generate-secrets + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + command: ["/bin/sh", "-c"] + args: + [ + "flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + containers: + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - serve + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: flyteadmin + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: 250m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + volumeMounts: + + - mountPath: /srv/flyte + name: shared-data + - mountPath: /etc/flyte/config + name: config-volume + - name: auth + mountPath: /etc/secrets/ + - command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: Flyte Admin OpenAPI + - name: SPEC_URL + value: /api/v1/openapi + - name: PORT + value: "8087" + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + name: redoc + ports: + - containerPort: 8087 + resources: + limits: + cpu: "0.1" + memory: 200Mi + serviceAccountName: flyteadmin + volumes: + + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-admin-config + name: config-volume + - configMap: + name: clusterresource-template + name: resource-templates + - name: auth + secret: + secretName: flyte-admin-auth +--- +# Source: flyte/templates/console/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteconsole + namespace: default + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "a14c2477f002d1392494a730c37ba7109e61df5cd417d431efa36841ee56bb9" + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "ghcr.io/flyteorg/flyteconsole:v0.20.0" + imagePullPolicy: "IfNotPresent" + name: flyteconsole + envFrom: + - configMapRef: + name: flyte-console-config + ports: + - containerPort: 8080 + resources: + limits: + cpu: 500m + memory: 250Mi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - mountPath: /srv/flyte + name: shared-data + volumes: + - emptyDir: {} + name: shared-data +--- +# Source: flyte/templates/datacatalog/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datacatalog + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "d63a2c1822443788cd796cf771ba9e4c722d7a93729e9f8f153b03e23e918da" + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + initContainers: + - command: + - datacatalog + - --config + - /etc/datacatalog/config/*.yaml + - migrate + - run + image: "ghcr.io/flyteorg/datacatalog:v0.3.4" + imagePullPolicy: "IfNotPresent" + name: run-migrations + volumeMounts: + + - mountPath: /etc/datacatalog/config + name: config-volume + containers: + - command: + - datacatalog + - --config + - /etc/datacatalog/config/*.yaml + - serve + image: "ghcr.io/flyteorg/datacatalog:v0.3.4" + imagePullPolicy: "IfNotPresent" + name: datacatalog + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: 500m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + volumeMounts: + + - mountPath: /etc/datacatalog/config + name: config-volume + serviceAccountName: datacatalog + volumes: + + - emptyDir: {} + name: shared-data + - configMap: + name: datacatalog-config + name: config-volume +--- +# Source: flyte/templates/minio/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: default + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte + template: + metadata: + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "minio/minio:RELEASE.2020-12-16T05-05-17Z" + imagePullPolicy: "IfNotPresent" + name: minio + args: + - server + - /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: miniostorage + ports: + - containerPort: 9000 + name: minio + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - name: minio-storage + mountPath: /var/lib/minioql/data + volumes: + - name: minio-storage + emptyDir: {} +--- +# Source: flyte/templates/postgres/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: default + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte + template: + metadata: + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "postgres:10.16" + imagePullPolicy: "IfNotPresent" + name: postgres + env: + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + ports: + - containerPort: 5432 + name: postgres + resources: + limits: + cpu: 1000m + memory: 512Mi + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - name: postgres-storage + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-storage + emptyDir: {} +--- +# Source: flyte/templates/propeller/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: default + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "7ce67c66483a6263afa5bed97db5cfb2bf546b52baf95e6880ea3f2d3a11f42" + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - command: + - flytepropeller + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: "ghcr.io/flyteorg/flytepropeller:v0.10.10" + imagePullPolicy: "IfNotPresent" + name: flytepropeller + ports: + - containerPort: 10254 + resources: + limits: + cpu: 200m + ephemeral-storage: 100Mi + memory: 200Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: auth + mountPath: /etc/secrets/ + serviceAccountName: flytepropeller + volumes: + - configMap: + name: flyte-propeller-config + name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create the actual deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-pod-webhook + namespace: default + labels: + app: flyte-pod-webhook +spec: + selector: + matchLabels: + app: flyte-pod-webhook + template: + metadata: + labels: + app: flyte-pod-webhook + app.kubernetes.io/name: flyte-pod-webhook + app.kubernetes.io/version: v0.10.10 + annotations: + configChecksum: "7ce67c66483a6263afa5bed97db5cfb2bf546b52baf95e6880ea3f2d3a11f42" + spec: + serviceAccountName: flyte-pod-webhook + initContainers: + - name: generate-secrets + image: "ghcr.io/flyteorg/flytepropeller:v0.10.10" + imagePullPolicy: "IfNotPresent" + command: + - flytepropeller + args: + - webhook + - init-certs + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + containers: + - name: webhook + image: "ghcr.io/flyteorg/flytepropeller:v0.10.10" + imagePullPolicy: "IfNotPresent" + command: + - flytepropeller + args: + - webhook + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + readOnly: true + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: config-volume + configMap: + name: flyte-propeller-config + - name: webhook-certs + secret: + secretName: flyte-pod-webhook +--- +# Source: flyte/templates/pytorch-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pytorch-operator + namespace: pytorch-operator + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + template: + metadata: + labels: + app.kubernetes.io/name: pytorch-operator + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "gcr.io/kubeflow-images-public/pytorch-operator:v1.0.0-g047cf0f" + imagePullPolicy: "IfNotPresent" + name: pytorch-operator + command: + - /pytorch-operator.v1 + - --alsologtostderr + - -v=1 + - --monitoring-port=8443 + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + resources: + limits: + cpu: 500m + memory: 1000M + requests: + cpu: 10m + memory: 50M + serviceAccountName: pytorch-operator +--- +# Source: flyte/templates/redis/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: default + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte + serviceName: redis-resource-manager + template: + metadata: + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "docker.io/bitnami/redis:4.0.2-r1" + imagePullPolicy: "IfNotPresent" + name: redis + env: + - name: REDIS_PASSWORD + value: mypassword + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - containerPort: 6379 + name: redis + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - mountPath: /bitnami + name: redis-data + + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data +--- +# Source: flyte/templates/admin/cronjob.yaml +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: syncresources + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + schedule: '*/1 * * * *' + jobTemplate: + spec: + template: + spec: + containers: + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - clusterresource + - sync + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: sync-cluster-resources + volumeMounts: + + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: OnFailure + serviceAccountName: flyteadmin + volumes: + + - configMap: + name: clusterresource-template + name: resource-templates + - configMap: + name: flyte-admin-config + name: config-volume +--- +# Source: flyte/templates/common/ingress.yaml +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: flyte + namespace: default +spec: + rules: + - http: + paths: + # This is useful only for frontend development + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 87 + # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml + - path: /console + pathType: ImplementationSpecific + backend: + serviceName: flyteconsole + servicePort: 80 + - path: /console/* + pathType: ImplementationSpecific + backend: + serviceName: flyteconsole + servicePort: 80 + - path: /api + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /api/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /healthcheck + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /v1/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /.well-known + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /.well-known/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /login + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /login/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /logout + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /logout/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /callback + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /callback/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /me + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /config + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /config/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. + - path: /flyteidl.service.AdminService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AdminService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 +--- +# Source: flyte/charts/contour/templates/certgen/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-contour-contour-certgen + namespace: default + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen +--- +# Source: flyte/charts/contour/templates/certgen/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: flyte-contour-contour-certgen + namespace: default + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update +--- +# Source: flyte/charts/contour/templates/certgen/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: flyte-contour-contour-certgen + namespace: default + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: flyte-contour-contour-certgen +subjects: + - kind: ServiceAccount + name: flyte-contour-contour-certgen +--- +# Source: flyte/charts/contour/templates/certgen/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: flyte-contour-contour-certgen + namespace: default + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen +spec: + ttlSecondsAfterFinished: 0 + template: + metadata: + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen + spec: + + containers: + - name: contour + image: docker.io/bitnami/contour:1.12.0-debian-10-r0 + imagePullPolicy: IfNotPresent + command: + - contour + args: + - certgen + - --kube + - --incluster + - --overwrite + - --secrets-format=compact + - --namespace=$(CONTOUR_NAMESPACE) + env: + - name: CONTOUR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + restartPolicy: Never + serviceAccountName: flyte-contour-contour-certgen + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: + runAsNonRoot: true + parallelism: 1 + completions: 1 + backoffLimit: 1 diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml new file mode 100644 index 0000000000..25432e99c2 --- /dev/null +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -0,0 +1,4200 @@ +--- +# Source: flyte/charts/contour/templates/contour/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-contour-contour + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour +--- +# Source: flyte/charts/contour/templates/envoy/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-contour-envoy + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: envoy +--- +# Source: flyte/charts/kubernetes-dashboard/templates/serviceaccount.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm + name: flyte-kubernetes-dashboard +--- +# Source: flyte/charts/sparkoperator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/charts/sparkoperator/templates/spark-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-spark + namespace: default + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/admin/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyteadmin + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/datacatalog/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: datacatalog + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/propeller/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flytepropeller + namespace: default + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create a Service Account for webhook +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-pod-webhook + namespace: default +--- +# Source: flyte/charts/kubernetes-dashboard/templates/secret.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# kubernetes-dashboard-certs +apiVersion: v1 +kind: Secret +metadata: + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm + name: flyte-kubernetes-dashboard-certs +type: Opaque +--- +# Source: flyte/charts/kubernetes-dashboard/templates/secret.yaml +# kubernetes-dashboard-csrf +apiVersion: v1 +kind: Secret +metadata: + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm + name: kubernetes-dashboard-csrf +type: Opaque +--- +# Source: flyte/charts/kubernetes-dashboard/templates/secret.yaml +# kubernetes-dashboard-key-holder +apiVersion: v1 +kind: Secret +metadata: + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm + name: kubernetes-dashboard-key-holder +type: Opaque +--- +# Source: flyte/templates/admin/deployment.yaml +apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- +# Source: flyte/templates/propeller/deployment.yaml +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: default +type: Opaque +stringData: + client_secret: foobar +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create an empty secret that the first propeller pod will populate +apiVersion: v1 +kind: Secret +metadata: + name: flyte-pod-webhook + namespace: default +type: Opaque +--- +# Source: flyte/charts/contour/templates/contour/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-contour + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour +data: + contour.yaml: | + accesslog-format: envoy + disablePermitInsecure: false + envoy-service-name: 'flyte-contour-envoy' + leaderelection: + configmap-namespace: 'default' + tls: {} +--- +# Source: flyte/charts/kubernetes-dashboard/templates/configmap.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm + name: kubernetes-dashboard-settings +data: +--- +# Source: flyte/templates/admin/cluster_resource_configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: clusterresource-template + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + aa_namespace.yaml: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes + + ab_project_resource_quota.yaml: | + apiVersion: v1 + kind: ResourceQuota + metadata: + name: project-quota + namespace: {{ namespace }} + spec: + hard: + limits.cpu: {{ projectQuotaCpu }} + limits.memory: {{ projectQuotaMemory }} + + ac_project_copilot_dataconfig.yaml: | + kind: ConfigMap + apiVersion: v1 + metadata: + name: flyte-data-config + namespace: {{ namespace }} + data: + config.yaml: | + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + type: minio + container: my-s3-bucket + enable-multicontainer: true +--- +# Source: flyte/templates/admin/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-admin-config + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + db.yaml: | + database: + dbname: flyte_development + host: postgres + port: 5432 + username: postgres + domain.yaml: | + domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + logger.yaml: | + logger: + level: 4 + show-source: true + server.yaml: | + auth: + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + authorizedUris: + - https://localhost:30081 + - http://flyteadmin:80 + - http://flyteadmin.flyte.svc.cluster.local:80 + userAuth: + openId: + baseUrl: https://accounts.google.com + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com + scopes: + - profile + - openid + flyteadmin: + eventVersion: 1 + metadataStoragePrefix: + - metadata + - admin + metricsScope: 'flyte:' + profilerPort: 10254 + roleNameKey: iam.amazonaws.com/role + testing: + host: http://flyteadmin + server: + grpcPort: 8089 + httpPort: 8088 + security: + allowCors: true + allowedHeaders: + - Content-Type + - flyte-authorization + allowedOrigins: + - '*' + secure: false + useAuth: false + remoteData.yaml: | + remoteData: + region: us-east-1 + scheme: local + signedUrls: + durationMinutes: 3 + storage.yaml: | + storage: + type: minio + container: "my-s3-bucket" + connection: + access-key: minio + auth-type: accesskey + secret-key: miniostorage + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + limits: + maxDownloadMBs: 10 + task_resource_defaults.yaml: | + task_resources: + defaults: + cpu: 100m + memory: 200Mi + storage: 5Mi + limits: + cpu: 2 + gpu: 1 + memory: 8Gi + storage: 20Mi + cluster_resources.yaml: | + cluster_resources: + customData: + - production: + - projectQuotaCpu: + value: "5" + - projectQuotaMemory: + value: 4000Mi + - staging: + - projectQuotaCpu: + value: "2" + - projectQuotaMemory: + value: 3000Mi + - development: + - projectQuotaCpu: + value: "4" + - projectQuotaMemory: + value: 3000Mi + refresh: 5m + templatePath: /etc/flyte/clusterresource/templates +--- +# Source: flyte/templates/console/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-console-config + namespace: default + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + BASE_URL: /console + CONFIG_DIR: /etc/flyte/config + DISABLE_AUTH: "1" +--- +# Source: flyte/templates/datacatalog/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: datacatalog-config + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + db.yaml: | + database: + dbname: flyte_development + host: postgres + port: 5432 + username: postgres + logger.yaml: | + logger: + level: 4 + show-source: true + server.yaml: | + application: + grpcPort: 8089 + grpcServerReflection: true + httpPort: 8080 + datacatalog: + metrics-scope: datacatalog + profiler-port: 10254 + storage-prefix: metadata/datacatalog + storage.yaml: | + storage: + type: minio + container: "my-s3-bucket" + connection: + access-key: minio + auth-type: accesskey + secret-key: miniostorage + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + limits: + maxDownloadMBs: 10 +--- +# Source: flyte/templates/propeller/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-propeller-config + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +data: + admin.yaml: | + admin: + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret + endpoint: flyteadmin:81 + insecure: true + event: + capacity: 1000 + rate: 500 + type: admin + catalog.yaml: | + catalog-cache: + endpoint: datacatalog:89 + insecure: true + type: datacatalog + copilot.yaml: | + plugins: + k8s: + co-pilot: + image: ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2 + name: flyte-copilot- + start-timeout: 30s + core.yaml: | + propeller: + downstream-eval-duration: 30s + enable-admin-launcher: true + leader-election: + enabled: true + lease-duration: 15s + lock-config-map: + name: propeller-leader + namespace: flyte + renew-deadline: 10s + retry-period: 2s + limit-namespace: all + max-workflow-retries: 30 + metadata-prefix: metadata/propeller + metrics-prefix: flyte + prof-port: 10254 + queue: + batch-size: -1 + batching-interval: 2s + queue: + base-delay: 5s + capacity: 1000 + max-delay: 120s + rate: 100 + type: maxof + sub-queue: + capacity: 100 + rate: 10 + type: bucket + type: batch + rawoutput-prefix: s3://my-s3-bucket/ + workers: 20 + workflow-reeval-duration: 30s + webhook: + certDir: /etc/webhook/certs + serviceName: flyte-pod-webhook + enabled_plugins.yaml: | + tasks: + task-plugins: + default-for-task-types: + container: container + container_array: k8s-array + sidecar: sidecar + enabled-plugins: + - container + - sidecar + - k8s-array + k8s.yaml: | + plugins: + k8s: + default-cpus: 100m + default-env-vars: + - FLYTE_AWS_ENDPOINT: http://minio.flyte:9000 + - FLYTE_AWS_ACCESS_KEY_ID: minio + - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage + default-memory: 200Mi + logger.yaml: | + logger: + level: 4 + show-source: true + resource_manager.yaml: | + propeller: + resourcemanager: + resourceMaxQuota: 10000 + type: noop + storage.yaml: | + storage: + type: minio + container: "my-s3-bucket" + connection: + access-key: minio + auth-type: accesskey + secret-key: miniostorage + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + limits: + maxDownloadMBs: 10 + cache.yaml: | + cache: + max_size_mbs: 0 + target_gc_percent: 70 + task_logs.yaml: | + plugins: + logs: + cloudwatch-enabled: false + kubernetes-enabled: true + kubernetes-template-uri: http://localhost:30082/#/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace }} +--- +# Source: flyte/charts/contour/templates/00-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.9 + creationTimestamp: null + name: extensionservices.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: ExtensionService + listKind: ExtensionServiceList + plural: extensionservices + shortNames: + - extensionservice + - extensionservices + singular: extensionservice + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ExtensionService is the schema for the Contour extension services API. An ExtensionService resource binds a network service to the Contour API so that Contour API features can be implemented by collaborating components. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. + properties: + loadBalancerPolicy: + description: The policy for load balancing GRPC service requests. Note that the `Cookie` load balancing strategy cannot be used here. + properties: + strategy: + description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. + type: string + type: object + protocol: + description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. + enum: + - h2 + - h2c + type: string + protocolVersion: + description: This field sets the version of the GRPC protocol that Envoy uses to send requests to the extension service. Since Contour always uses the v3 Envoy API, this is currently fixed at "v3". However, other protocol options will be available in future. + enum: + - v3 + type: string + services: + description: Services specifies the set of Kubernetes Service resources that receive GRPC extension API requests. If no weights are specified for any of the entries in this array, traffic will be spread evenly across all the services. Otherwise, traffic is balanced proportionally to the Weight field in each entry. + items: + description: ExtensionServiceTarget defines an Kubernetes Service to target with extension service traffic. + properties: + name: + description: Name is the name of Kubernetes service that will accept service traffic. + type: string + port: + description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. + exclusiveMaximum: true + maximum: 65536 + minimum: 1 + type: integer + weight: + description: Weight defines proportion of traffic to balance to the Kubernetes Service. + format: int32 + type: integer + required: + - name + - port + type: object + minItems: 1 + type: array + timeoutPolicy: + description: The timeout policy for requests to the services. + properties: + idle: + description: Timeout after which, if there are no active requests for this route, the connection between Envoy and the backend or Envoy and the external client will be closed. If not specified, there is no per-route idle timeout, though a connection manager-wide stream_idle_timeout default of 5m still applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + response: + description: Timeout for receiving a response from the server after processing a request from client. If not supplied, Envoy's default value of 15s applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + type: object + validation: + description: UpstreamValidation defines how to verify the backend service's certificate + properties: + caSecret: + description: Name of the Kubernetes secret be used to validate the certificate presented by the backend + type: string + subjectName: + description: Key which is expected to be present in the 'subjectAltName' of the presented certificate + type: string + required: + - caSecret + - subjectName + type: object + required: + - services + type: object + status: + description: ExtensionServiceStatus defines the observed state of an ExtensionService resource. + properties: + conditions: + description: "Conditions contains the current status of the ExtensionService resource. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. \n Contour will not modify any other Conditions set in this block, in case some other controller wants to add a Condition." + items: + description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." + properties: + errors: + description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: flyte/charts/contour/templates/00-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.9 + creationTimestamp: null + name: httpproxies.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: HTTPProxy + listKind: HTTPProxyList + plural: httpproxies + shortNames: + - proxy + - proxies + singular: httpproxy + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Fully qualified domain name + jsonPath: .spec.virtualhost.fqdn + name: FQDN + type: string + - description: Secret with TLS credentials + jsonPath: .spec.virtualhost.tls.secretName + name: TLS Secret + type: string + - description: The current status of the HTTPProxy + jsonPath: .status.currentStatus + name: Status + type: string + - description: Description of the current status + jsonPath: .status.description + name: Status Description + type: string + name: v1 + schema: + openAPIV3Schema: + description: HTTPProxy is an Ingress CRD specification. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HTTPProxySpec defines the spec of the CRD. + properties: + ggvar: + description: Testing new param for CRD + type: string + includes: + description: Includes allow for specific routing configuration to be included from another HTTPProxy, possibly in another namespace. + items: + description: Include describes a set of policies that can be applied to an HTTPProxy in a namespace. + properties: + conditions: + description: 'Conditions are a set of rules that are applied to included HTTPProxies. In effect, they are added onto the Conditions of included HTTPProxy Route structs. When applied, they are merged using AND, with one exception: There can be only one Prefix MatchCondition per Conditions slice. More than one Prefix, or contradictory Conditions, will make the include invalid.' + items: + description: MatchCondition are a general holder for matching rules for HTTPProxies. One of Prefix or Header must be provided. + properties: + header: + description: Header specifies the header condition to match. + properties: + contains: + description: Contains specifies a substring that must be present in the header value. + type: string + exact: + description: Exact specifies a string that the header value must be equal to. + type: string + name: + description: Name is the name of the header to match against. Name is required. Header names are case insensitive. + type: string + notcontains: + description: NotContains specifies a substring that must not be present in the header value. + type: string + notexact: + description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value. + type: string + present: + description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent. + type: boolean + required: + - name + type: object + prefix: + description: Prefix defines a prefix match for a request. + type: string + type: object + type: array + name: + description: Name of the HTTPProxy + type: string + namespace: + description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. + type: string + required: + - name + type: object + type: array + routes: + description: Routes are the ingress routes. If TCPProxy is present, Routes is ignored. + items: + description: Route contains the set of routes for a virtual host. + properties: + authPolicy: + description: AuthPolicy updates the authorization policy that was set on the root HTTPProxy object for client requests that match this route. + properties: + context: + additionalProperties: + type: string + description: Context is a set of key/value pairs that are sent to the authentication server in the check request. If a context is provided at an enclosing scope, the entries are merged such that the inner scope overrides matching keys from the outer scope. + type: object + disabled: + description: When true, this field disables client request authentication for the scope of the policy. + type: boolean + type: object + conditions: + description: 'Conditions are a set of rules that are applied to a Route. When applied, they are merged using AND, with one exception: There can be only one Prefix MatchCondition per Conditions slice. More than one Prefix, or contradictory Conditions, will make the route invalid.' + items: + description: MatchCondition are a general holder for matching rules for HTTPProxies. One of Prefix or Header must be provided. + properties: + header: + description: Header specifies the header condition to match. + properties: + contains: + description: Contains specifies a substring that must be present in the header value. + type: string + exact: + description: Exact specifies a string that the header value must be equal to. + type: string + name: + description: Name is the name of the header to match against. Name is required. Header names are case insensitive. + type: string + notcontains: + description: NotContains specifies a substring that must not be present in the header value. + type: string + notexact: + description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value. + type: string + present: + description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent. + type: boolean + required: + - name + type: object + prefix: + description: Prefix defines a prefix match for a request. + type: string + type: object + type: array + enableWebsockets: + description: Enables websocket support for the route. + type: boolean + healthCheckPolicy: + description: The health check policy for this route. + properties: + healthyThresholdCount: + description: The number of healthy health checks required before a host is marked healthy + format: int64 + minimum: 0 + type: integer + host: + description: The value of the host header in the HTTP health check request. If left empty (default value), the name "contour-envoy-healthcheck" will be used. + type: string + intervalSeconds: + description: The interval (seconds) between health checks + format: int64 + type: integer + path: + description: HTTP endpoint used to perform health checks on upstream service + type: string + timeoutSeconds: + description: The time to wait (seconds) for a health check response + format: int64 + type: integer + unhealthyThresholdCount: + description: The number of unhealthy health checks required before a host is marked unhealthy + format: int64 + minimum: 0 + type: integer + required: + - path + type: object + loadBalancerPolicy: + description: The load balancing policy for this route. + properties: + strategy: + description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. + type: string + type: object + pathRewritePolicy: + description: The policy for rewriting the path of the request URL after the request has been routed to a Service. + properties: + replacePrefix: + description: ReplacePrefix describes how the path prefix should be replaced. + items: + description: ReplacePrefix describes a path prefix replacement. + properties: + prefix: + description: "Prefix specifies the URL path prefix to be replaced. \n If Prefix is specified, it must exactly match the MatchCondition prefix that is rendered by the chain of including HTTPProxies and only that path prefix will be replaced by Replacement. This allows HTTPProxies that are included through multiple roots to only replace specific path prefixes, leaving others unmodified. \n If Prefix is not specified, all routing prefixes rendered by the include chain will be replaced." + minLength: 1 + type: string + replacement: + description: Replacement is the string that the routing path prefix will be replaced with. This must not be empty. + minLength: 1 + type: string + required: + - replacement + type: object + type: array + type: object + permitInsecure: + description: Allow this path to respond to insecure requests over HTTP which are normally not permitted when a `virtualhost.tls` block is present. + type: boolean + requestHeadersPolicy: + description: The policy for managing request headers during proxying. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + responseHeadersPolicy: + description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + retryPolicy: + description: The retry policy for this route. + properties: + count: + description: NumRetries is maximum allowed number of retries. If not supplied, the number of retries is one. + format: int64 + minimum: 0 + type: integer + perTryTimeout: + description: PerTryTimeout specifies the timeout per retry attempt. Ignored if NumRetries is not supplied. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + retriableStatusCodes: + description: "RetriableStatusCodes specifies the HTTP status codes that should be retried. \n This field is only respected when you include `retriable-status-codes` in the `RetryOn` field." + items: + format: int32 + type: integer + type: array + retryOn: + description: "RetryOn specifies the conditions on which to retry a request. \n Supported [HTTP conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-on): \n - `5xx` - `gateway-error` - `reset` - `connect-failure` - `retriable-4xx` - `refused-stream` - `retriable-status-codes` - `retriable-headers` \n Supported [gRPC conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-grpc-on): \n - `cancelled` - `deadline-exceeded` - `internal` - `resource-exhausted` - `unavailable`" + items: + description: RetryOn is a string type alias with validation to ensure that the value is valid. + enum: + - 5xx + - gateway-error + - reset + - connect-failure + - retriable-4xx + - refused-stream + - retriable-status-codes + - retriable-headers + - cancelled + - deadline-exceeded + - internal + - resource-exhausted + - unavailable + type: string + type: array + type: object + services: + description: Services are the services to proxy traffic. + items: + description: Service defines an Kubernetes Service to proxy traffic. + properties: + mirror: + description: If Mirror is true the Service will receive a read only mirror of the traffic for this route. + type: boolean + name: + description: Name is the name of Kubernetes service to proxy traffic. Names defined here will be used to look up corresponding endpoints which contain the ips to route. + type: string + port: + description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. + exclusiveMaximum: true + maximum: 65536 + minimum: 1 + type: integer + protocol: + description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. + enum: + - h2 + - h2c + - tls + type: string + requestHeadersPolicy: + description: The policy for managing request headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + responseHeadersPolicy: + description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + validation: + description: UpstreamValidation defines how to verify the backend service's certificate + properties: + caSecret: + description: Name of the Kubernetes secret be used to validate the certificate presented by the backend + type: string + subjectName: + description: Key which is expected to be present in the 'subjectAltName' of the presented certificate + type: string + required: + - caSecret + - subjectName + type: object + weight: + description: Weight defines percentage of traffic to balance traffic + format: int64 + minimum: 0 + type: integer + required: + - name + - port + type: object + minItems: 1 + type: array + timeoutPolicy: + description: The timeout policy for this route. + properties: + idle: + description: Timeout after which, if there are no active requests for this route, the connection between Envoy and the backend or Envoy and the external client will be closed. If not specified, there is no per-route idle timeout, though a connection manager-wide stream_idle_timeout default of 5m still applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + response: + description: Timeout for receiving a response from the server after processing a request from client. If not supplied, Envoy's default value of 15s applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + type: object + required: + - services + type: object + type: array + tcpproxy: + description: TCPProxy holds TCP proxy information. + properties: + healthCheckPolicy: + description: The health check policy for this tcp proxy + properties: + healthyThresholdCount: + description: The number of healthy health checks required before a host is marked healthy + format: int32 + type: integer + intervalSeconds: + description: The interval (seconds) between health checks + format: int64 + type: integer + timeoutSeconds: + description: The time to wait (seconds) for a health check response + format: int64 + type: integer + unhealthyThresholdCount: + description: The number of unhealthy health checks required before a host is marked unhealthy + format: int32 + type: integer + type: object + include: + description: Include specifies that this tcpproxy should be delegated to another HTTPProxy. + properties: + name: + description: Name of the child HTTPProxy + type: string + namespace: + description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. + type: string + required: + - name + type: object + includes: + description: "IncludesDeprecated allow for specific routing configuration to be appended to another HTTPProxy in another namespace. \n Exists due to a mistake when developing HTTPProxy and the field was marked plural when it should have been singular. This field should stay to not break backwards compatibility to v1 users." + properties: + name: + description: Name of the child HTTPProxy + type: string + namespace: + description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. + type: string + required: + - name + type: object + loadBalancerPolicy: + description: The load balancing policy for the backend services. + properties: + strategy: + description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. + type: string + type: object + services: + description: Services are the services to proxy traffic + items: + description: Service defines an Kubernetes Service to proxy traffic. + properties: + mirror: + description: If Mirror is true the Service will receive a read only mirror of the traffic for this route. + type: boolean + name: + description: Name is the name of Kubernetes service to proxy traffic. Names defined here will be used to look up corresponding endpoints which contain the ips to route. + type: string + port: + description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. + exclusiveMaximum: true + maximum: 65536 + minimum: 1 + type: integer + protocol: + description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. + enum: + - h2 + - h2c + - tls + type: string + requestHeadersPolicy: + description: The policy for managing request headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + responseHeadersPolicy: + description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names to remove. + items: + type: string + type: array + set: + description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + validation: + description: UpstreamValidation defines how to verify the backend service's certificate + properties: + caSecret: + description: Name of the Kubernetes secret be used to validate the certificate presented by the backend + type: string + subjectName: + description: Key which is expected to be present in the 'subjectAltName' of the presented certificate + type: string + required: + - caSecret + - subjectName + type: object + weight: + description: Weight defines percentage of traffic to balance traffic + format: int64 + minimum: 0 + type: integer + required: + - name + - port + type: object + type: array + type: object + virtualhost: + description: Virtualhost appears at most once. If it is present, the object is considered to be a "root" HTTPProxy. + properties: + authorization: + description: This field configures an extension service to perform authorization for this virtual host. Authorization can only be configured on virtual hosts that have TLS enabled. If the TLS configuration requires client certificate /validation, the client certificate is always included in the authentication check request. + properties: + authPolicy: + description: AuthPolicy sets a default authorization policy for client requests. This policy will be used unless overridden by individual routes. + properties: + context: + additionalProperties: + type: string + description: Context is a set of key/value pairs that are sent to the authentication server in the check request. If a context is provided at an enclosing scope, the entries are merged such that the inner scope overrides matching keys from the outer scope. + type: object + disabled: + description: When true, this field disables client request authentication for the scope of the policy. + type: boolean + type: object + extensionRef: + description: ExtensionServiceRef specifies the extension resource that will authorize client requests. + properties: + apiVersion: + description: API version of the referent. If this field is not specified, the default "projectcontour.io/v1alpha1" will be used + minLength: 1 + type: string + name: + description: "Name of the referent. \n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" + minLength: 1 + type: string + namespace: + description: "Namespace of the referent. If this field is not specifies, the namespace of the resource that targets the referent will be used. \n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/" + minLength: 1 + type: string + type: object + failOpen: + description: If FailOpen is true, the client request is forwarded to the upstream service even if the authorization server fails to respond. This field should not be set in most cases. It is intended for use only while migrating applications from internal authorization to Contour external authorization. + type: boolean + responseTimeout: + description: ResponseTimeout configures maximum time to wait for a check response from the authorization server. Timeout durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". The string "infinity" is also a valid input and specifies no timeout. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + required: + - extensionRef + type: object + corsPolicy: + description: Specifies the cross-origin policy to apply to the VirtualHost. + properties: + allowCredentials: + description: Specifies whether the resource allows credentials. + type: boolean + allowHeaders: + description: AllowHeaders specifies the content for the *access-control-allow-headers* header. + items: + description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. + pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ + type: string + type: array + allowMethods: + description: AllowMethods specifies the content for the *access-control-allow-methods* header. + items: + description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. + pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ + type: string + type: array + allowOrigin: + description: AllowOrigin specifies the origins that will be allowed to do CORS requests. "*" means allow any origin. + items: + type: string + type: array + exposeHeaders: + description: ExposeHeaders Specifies the content for the *access-control-expose-headers* header. + items: + description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. + pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ + type: string + type: array + maxAge: + description: MaxAge indicates for how long the results of a preflight request can be cached. MaxAge durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Only positive values are allowed while 0 disables the cache requiring a preflight OPTIONS check for all cross-origin requests. + type: string + required: + - allowMethods + - allowOrigin + type: object + fqdn: + description: The fully qualified domain name of the root of the ingress tree all leaves of the DAG rooted at this object relate to the fqdn. + type: string + tls: + description: If present the fields describes TLS properties of the virtual host. The SNI names that will be matched on are described in fqdn, the tls.secretName secret must contain a certificate that itself contains a name that matches the FQDN. + properties: + clientValidation: + description: "ClientValidation defines how to verify the client certificate when an external client establishes a TLS connection to Envoy. \n This setting: \n 1. Enables TLS client certificate validation. 2. Requires clients to present a TLS certificate (i.e. not optional validation). 3. Specifies how the client certificate will be validated." + properties: + caSecret: + description: Name of a Kubernetes secret that contains a CA certificate bundle. The client certificate must validate against the certificates in the bundle. + minLength: 1 + type: string + required: + - caSecret + type: object + enableFallbackCertificate: + description: EnableFallbackCertificate defines if the vhost should allow a default certificate to be applied which handles all requests which don't match the SNI defined in this vhost. + type: boolean + minimumProtocolVersion: + description: Minimum TLS version this vhost should negotiate + type: string + passthrough: + description: Passthrough defines whether the encrypted TLS handshake will be passed through to the backing cluster. Either Passthrough or SecretName must be specified, but not both. + type: boolean + secretName: + description: SecretName is the name of a TLS secret in the current namespace. Either SecretName or Passthrough must be specified, but not both. If specified, the named secret must contain a matching certificate for the virtual host's FQDN. + type: string + type: object + required: + - fqdn + type: object + type: object + status: + description: Status is a container for computed information about the HTTPProxy. + properties: + conditions: + description: "Conditions contains information about the current status of the HTTPProxy, in an upstream-friendly container. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, and vice versa. \n Contour will leave untouched any other Conditions set in this block, in case some other controller wants to add a Condition. \n If you are another controller owner and wish to add a condition, you *should* namespace your condition with a label, like `controller.domain.com/ConditionName`." + items: + description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." + properties: + errors: + description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + currentStatus: + type: string + description: + type: string + loadBalancer: + description: LoadBalancer contains the current status of the load balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) + type: string + type: object + type: array + type: object + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: flyte/charts/contour/templates/00-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.9 + creationTimestamp: null + name: tlscertificatedelegations.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: TLSCertificateDelegation + listKind: TLSCertificateDelegationList + plural: tlscertificatedelegations + shortNames: + - tlscerts + singular: tlscertificatedelegation + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: TLSCertificateDelegation is an TLS Certificate Delegation CRD specification. See design/tls-certificate-delegation.md for details. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TLSCertificateDelegationSpec defines the spec of the CRD + properties: + delegations: + items: + description: CertificateDelegation maps the authority to reference a secret in the current namespace to a set of namespaces. + properties: + secretName: + description: required, the name of a secret in the current namespace. + type: string + targetNamespaces: + description: required, the namespaces the authority to reference the the secret will be delegated to. If TargetNamespaces is nil or empty, the CertificateDelegation' is ignored. If the TargetNamespace list contains the character, "*" the secret will be delegated to all namespaces. + items: + type: string + type: array + required: + - secretName + - targetNamespaces + type: object + type: array + required: + - delegations + type: object + status: + description: TLSCertificateDelegationStatus allows for the status of the delegation to be presented to the user. + properties: + conditions: + description: "Conditions contains information about the current status of the HTTPProxy, in an upstream-friendly container. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, and vice versa. \n Contour will leave untouched any other Conditions set in this block, in case some other controller wants to add a Condition. \n If you are another controller owner and wish to add a condition, you *should* namespace your condition with a label, like `controller.domain.com\\ConditionName`." + items: + description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." + properties: + errors: + description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." + items: + description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." + properties: + message: + description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." + maxLength: 32768 + type: string + reason: + description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: flyte/templates/propeller/crds/flyteworkflow.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: flyteworkflows.flyte.lyft.com +spec: + group: flyte.lyft.com + names: + kind: FlyteWorkflow + plural: flyteworkflows + shortNames: + - fly + singular: flyteworkflow + scope: Namespaced + version: v1alpha1 +--- +# Source: flyte/charts/contour/templates/contour/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyte-contour-contour + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - update + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - apiGroups: + - networking.k8s.io + resources: + - gatewayclasses + - gateways + - httproutes + - tcproutes + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - create + - get + - update + - apiGroups: + - projectcontour.io + resources: + - extensionservices + verbs: + - get + - list + - watch + - apiGroups: + - projectcontour.io + resources: + - extensionservices/status + verbs: + - create + - get + - update + - apiGroups: + - projectcontour.io + resources: + - httpproxies + - tlscertificatedelegations + verbs: + - get + - list + - watch + - apiGroups: + - projectcontour.io + resources: + - httpproxies/status + verbs: + - create + - get + - update +--- +# Source: flyte/charts/kubernetes-dashboard/templates/clusterrole-metrics.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: "flyte-kubernetes-dashboard-metrics" + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm +rules: + # Allow Metrics Scraper to get metrics from the Metrics server + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] +--- +# Source: flyte/charts/sparkoperator/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - "*" +- apiGroups: + - "" + resources: + - services + - configmaps + - secrets + verbs: + - create + - get + - delete + - update +- apiGroups: + - extensions + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - get + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - patch +- apiGroups: + - "" + resources: + - resourcequotas + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - update + - delete +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - create + - get + - update + - delete +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + - sparkapplications/status + - scheduledsparkapplications + - scheduledsparkapplications/status + verbs: + - "*" +--- +# Source: flyte/templates/admin/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyteadmin + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + - flyte.lyft.com + - rbac.authorization.k8s.io + resources: + - configmaps + - flyteworkflows + - namespaces + - pods + - resourcequotas + - roles + - rolebindings + - secrets + - services + - serviceaccounts + - spark-role + verbs: + - '*' +--- +# Source: flyte/templates/propeller/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flytepropeller + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +rules: +# Allow RO access to PODS +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +# Allow Event recording access +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - delete + - patch +# Allow Access All plugin objects +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +# Allow Access to CRD +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete + - update +# Allow Access to all resources under flyte.lyft.com +- apiGroups: + - flyte.lyft.com + resources: + - flyteworkflows + - flyteworkflows/finalizers + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - post + - deletecollection +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create a ClusterRole for the webhook +# https://kubernetes.io/docs/admin/authorization/rbac/ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flyte-pod-webhook + namespace: default +rules: + - apiGroups: + - "*" + resources: + - mutatingwebhookconfigurations + - secrets + - pods + verbs: + - get + - create + - update + - patch +--- +# Source: flyte/charts/contour/templates/contour/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flyte-contour-contour + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-contour-contour +subjects: + - kind: ServiceAccount + name: flyte-contour-contour + namespace: default +--- +# Source: flyte/charts/kubernetes-dashboard/templates/clusterrolebinding-metrics.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: "flyte-kubernetes-dashboard-metrics" + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-kubernetes-dashboard-metrics +subjects: + - kind: ServiceAccount + name: flyte-kubernetes-dashboard + namespace: default +--- +# Source: flyte/charts/sparkoperator/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +subjects: + - kind: ServiceAccount + name: flyte-sparkoperator + namespace: default +roleRef: + kind: ClusterRole + name: flyte-sparkoperator + apiGroup: rbac.authorization.k8s.io +--- +# Source: flyte/templates/admin/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flyteadmin-binding + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyteadmin +subjects: +- kind: ServiceAccount + name: flyteadmin + namespace: default +--- +# Source: flyte/templates/propeller/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flytepropeller + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flytepropeller +subjects: +- kind: ServiceAccount + name: flytepropeller + namespace: default +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create a binding from Role -> ServiceAccount +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flyte-pod-webhook + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-pod-webhook +subjects: + - kind: ServiceAccount + name: flyte-pod-webhook + namespace: default +--- +# Source: flyte/charts/kubernetes-dashboard/templates/role.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: flyte-kubernetes-dashboard + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm +rules: + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + # Allow Dashboard to get metrics. + - apiGroups: [""] + resources: ["services"] + resourceNames: ["heapster", "dashboard-metrics-scraper"] + verbs: ["proxy"] + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"] + verbs: ["get"] +--- +# Source: flyte/charts/sparkoperator/templates/spark-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: spark-role + namespace: default + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - "*" +- apiGroups: + - "" + resources: + - services + verbs: + - "*" +--- +# Source: flyte/charts/kubernetes-dashboard/templates/rolebinding.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: flyte-kubernetes-dashboard + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: flyte-kubernetes-dashboard +subjects: + - kind: ServiceAccount + name: flyte-kubernetes-dashboard + namespace: default +--- +# Source: flyte/charts/sparkoperator/templates/spark-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spark + namespace: default + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +subjects: +- kind: ServiceAccount + name: flyte-spark + namespace: default +roleRef: + kind: Role + name: spark-role + apiGroup: rbac.authorization.k8s.io +--- +# Source: flyte/charts/contour/templates/contour/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyte-contour + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour +spec: + ports: + - port: 8001 + name: xds + protocol: TCP + targetPort: 8001 + selector: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: contour + type: ClusterIP +--- +# Source: flyte/charts/contour/templates/envoy/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyte-contour-envoy + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: envoy + annotations: + # This annotation puts the AWS ELB into "TCP" mode so that it does not + # do HTTP negotiation for HTTPS connections at the ELB edge. + # The downside of this is the remote IP address of all connections will + # appear to be the internal address of the ELB. See docs/proxy-proto.md + # for information about enabling the PROXY protocol on the ELB to recover + # the original remote IP address. + # We don't set this for nlb, per the contour docs. + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp +spec: + externalTrafficPolicy: "Local" + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + nodePort: 30081 + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: envoy + type: NodePort +--- +# Source: flyte/charts/kubernetes-dashboard/templates/service.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: flyte-kubernetes-dashboard + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kubernetes-dashboard + kubernetes.io/cluster-service: "true" +spec: + type: NodePort + ports: + - port: 30082 + targetPort: http + name: http + selector: + app.kubernetes.io/name: kubernetes-dashboard + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: kubernetes-dashboard +--- +# Source: flyte/templates/admin/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyteadmin + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + annotations: + projectcontour.io/upstream-protocol.h2c: grpc +spec: + type: ClusterIP + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 81 + protocol: TCP + targetPort: 8089 + - name: redoc + protocol: TCP + port: 87 + targetPort: 8087 + - name: http-metrics + protocol: TCP + port: 10254 + selector: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/console/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flyteconsole + namespace: default + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/datacatalog/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: datacatalog + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + annotations: + projectcontour.io/upstream-protocol.h2c: grpc +spec: + type: NodePort + ports: + - name: grpc-2 + port: 8089 + protocol: TCP + targetPort: 8089 + - name: http + port: 88 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 89 + protocol: TCP + targetPort: 8089 + selector: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/minio/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: default + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: minio + port: 9000 + protocol: TCP + targetPort: minio + selector: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/postgres/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: default + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: postgres + port: 5432 + protocol: TCP + targetPort: postgres + selector: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte +--- +# Source: flyte/templates/propeller/webhook.yaml +# Service +apiVersion: v1 +kind: Service +metadata: + name: flyte-pod-webhook + namespace: default + annotations: + projectcontour.io/upstream-protocol.h2c: grpc +spec: + selector: + app: flyte-pod-webhook + ports: + - name: https + protocol: TCP + port: 443 + targetPort: 9443 +--- +# Source: flyte/charts/contour/templates/envoy/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: flyte-contour-envoy + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: envoy +spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: envoy + template: + metadata: + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: envoy + spec: + + affinity: + podAffinity: + + podAntiAffinity: + + nodeAffinity: + + terminationGracePeriodSeconds: 300 + hostNetwork: false + dnsPolicy: ClusterFirst + containers: + - command: + - contour + args: + - envoy + - shutdown-manager + image: docker.io/bitnami/contour:1.12.0-debian-10-r0 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - contour + - envoy + - shutdown + livenessProbe: + httpGet: + path: /healthz + port: 8090 + initialDelaySeconds: 120 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + name: shutdown-manager + - command: + - envoy + args: + - -c + - /config/envoy.json + - --service-cluster $(CONTOUR_NAMESPACE) + - --service-node $(ENVOY_POD_NAME) + - --log-level info + image: docker.io/bitnami/envoy:1.16.2-debian-10-r54 + imagePullPolicy: IfNotPresent + name: envoy + securityContext: + runAsUser: 0 + env: + - name: CONTOUR_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + ports: + - containerPort: 80 + hostPort: 80 + name: http + protocol: TCP + - containerPort: 443 + hostPort: 443 + name: https + protocol: TCP + - containerPort: 8002 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: 8002 + initialDelaySeconds: 10 + periodSeconds: 3 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /ready + port: 8002 + initialDelaySeconds: 120 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - name: envoy-config + mountPath: /config + - name: envoycert + mountPath: /certs + lifecycle: + preStop: + httpGet: + path: /shutdown + port: 8090 + scheme: HTTP + initContainers: + - command: + - contour + args: + - bootstrap + - /config/envoy.json + - --xds-address=flyte-contour + - --xds-port=8001 + - --resources-dir=/config/resources + - --envoy-cafile=/certs/ca.crt + - --envoy-cert-file=/certs/tls.crt + - --envoy-key-file=/certs/tls.key + image: docker.io/bitnami/contour:1.12.0-debian-10-r0 + imagePullPolicy: IfNotPresent + name: envoy-initconfig + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - name: envoy-config + mountPath: /config + - name: envoycert + mountPath: /certs + readOnly: true + env: + - name: CONTOUR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + automountServiceAccountToken: false + serviceAccountName: flyte-contour-envoy + volumes: + - name: envoy-config + emptyDir: {} + - name: envoycert + secret: + secretName: envoycert + restartPolicy: Always +--- +# Source: flyte/charts/contour/templates/contour/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-contour-contour + namespace: default + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # This value of maxSurge means that during a rolling update + # the new ReplicaSet will be created first. + maxSurge: 50% + selector: + matchLabels: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: contour + template: + metadata: + annotations: + checksum/config: 1c6f180e2479c6aff9159e6eea066d5238a8c9ee1f38f19a6de2f070a88e5ad9 + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour + spec: + + affinity: + podAffinity: + + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: contour + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: contour + namespaces: + - "default" + topologyKey: kubernetes.io/hostname + weight: 1 + nodeAffinity: + + containers: + - command: + - contour + args: + - serve + - --incluster + - --xds-address=0.0.0.0 + - --xds-port=8001 + - --envoy-service-http-port=80 + - --envoy-service-https-port=443 + - --contour-cafile=/certs/ca.crt + - --contour-cert-file=/certs/tls.crt + - --contour-key-file=/certs/tls.key + - --config-path=/config/contour.yaml + image: docker.io/bitnami/contour:1.12.0-debian-10-r0 + imagePullPolicy: IfNotPresent + name: contour + ports: + - containerPort: 8001 + name: xds + protocol: TCP + - containerPort: 8000 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8000 + initialDelaySeconds: 120 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + readinessProbe: + tcpSocket: + port: 8001 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - name: contourcert + mountPath: /certs + readOnly: true + - name: contour-config + mountPath: /config + readOnly: true + env: + - name: CONTOUR_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + dnsPolicy: ClusterFirst + serviceAccountName: flyte-contour-contour + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: + runAsNonRoot: true + volumes: + - name: contourcert + secret: + secretName: contourcert + - name: contour-config + configMap: + name: flyte-contour + defaultMode: 0644 + items: + - key: contour.yaml + path: contour.yaml +--- +# Source: flyte/charts/kubernetes-dashboard/templates/deployment.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-kubernetes-dashboard + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kubernetes-dashboard +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: kubernetes-dashboard + app.kubernetes.io/instance: flyte + app.kubernetes.io/component: kubernetes-dashboard + template: + metadata: + annotations: + seccomp.security.alpha.kubernetes.io/pod: runtime/default + labels: + app.kubernetes.io/name: kubernetes-dashboard + helm.sh/chart: kubernetes-dashboard-4.0.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "2.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kubernetes-dashboard + spec: + serviceAccountName: flyte-kubernetes-dashboard + containers: + - name: kubernetes-dashboard + image: "kubernetesui/dashboard:v2.2.0" + imagePullPolicy: IfNotPresent + args: + - --namespace=default + - --metrics-provider=none + - --enable-skip-login + - --enable-insecure-login + - --disable-settings-authorizer + ports: + - name: http + containerPort: 9090 + protocol: TCP + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs + # Create on-disk volume to store exec logs + - mountPath: /tmp + name: tmp-volume + livenessProbe: + httpGet: + scheme: HTTP + path: / + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 + resources: + limits: + cpu: 2 + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 2001 + runAsUser: 1001 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: flyte-kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} +--- +# Source: flyte/charts/sparkoperator/templates/deployment.yaml +# If the admission webhook is enabled, then a post-install step is required +# to generate and install the secret in the operator namespace. + +# In the post-install hook, the token corresponding to the operator service account +# is used to authenticate with the Kubernetes API server to install the secret bundle. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-sparkoperator + labels: + helm.sh/chart: sparkoperator-1.0.6 + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + app.kubernetes.io/version: "v1beta2-1.2.0-3.0.0" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + strategy: + type: Recreate + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" + prometheus.io/path: /metrics + labels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/instance: flyte + spec: + serviceAccountName: flyte-sparkoperator + securityContext: + {} + containers: + - name: sparkoperator + image: gcr.io/spark-operator/spark-operator:v1beta2-1.2.0-3.0.0 + imagePullPolicy: IfNotPresent + securityContext: + {} + ports: + - name: "metrics" + containerPort: 10254 + + args: + - -v=2 + - -logtostderr + - -namespace= + - -ingress-url-format= + - -controller-threads=10 + - -resync-interval=30 + - -enable-batch-scheduler=false + - -enable-metrics=true + - -metrics-labels=app_type + - -metrics-port=10254 + - -metrics-endpoint=/metrics + - -metrics-prefix= + - -enable-resource-quota-enforcement=false + resources: + limits: + cpu: 1000m + memory: 500M + requests: + cpu: 10m + memory: 50M +--- +# Source: flyte/templates/admin/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteadmin + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "8ad93c5353943acd48572ff9db564f0a8544904593c5eee727204420a4b4239" + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + initContainers: + - name: check-db-ready + image: postgres:10.16-alpine + command: + - sh + - -c + - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done; + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - migrate + - run + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: run-migrations + volumeMounts: + + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - migrate + - seed-projects + - flytesnacks + - flytetester + - flyteexamples + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: seed-projects + volumeMounts: + + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - clusterresource + - sync + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: sync-cluster-resources + volumeMounts: + + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + - name: generate-secrets + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + command: ["/bin/sh", "-c"] + args: + [ + "flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + containers: + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - serve + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: flyteadmin + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: 250m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + volumeMounts: + + - mountPath: /srv/flyte + name: shared-data + - mountPath: /etc/flyte/config + name: config-volume + - name: auth + mountPath: /etc/secrets/ + - command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: Flyte Admin OpenAPI + - name: SPEC_URL + value: /api/v1/openapi + - name: PORT + value: "8087" + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + name: redoc + ports: + - containerPort: 8087 + resources: + limits: + cpu: "0.1" + memory: 200Mi + serviceAccountName: flyteadmin + volumes: + + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-admin-config + name: config-volume + - configMap: + name: clusterresource-template + name: resource-templates + - name: auth + secret: + secretName: flyte-admin-auth +--- +# Source: flyte/templates/console/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteconsole + namespace: default + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "a14c2477f002d1392494a730c37ba7109e61df5cd417d431efa36841ee56bb9" + labels: + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "ghcr.io/flyteorg/flyteconsole:v0.20.0" + imagePullPolicy: "IfNotPresent" + name: flyteconsole + envFrom: + - configMapRef: + name: flyte-console-config + ports: + - containerPort: 8080 + resources: + limits: + cpu: 500m + memory: 250Mi + requests: + cpu: 10m + memory: 50Mi + volumeMounts: + - mountPath: /srv/flyte + name: shared-data + volumes: + - emptyDir: {} + name: shared-data +--- +# Source: flyte/templates/datacatalog/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datacatalog + namespace: default + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "4077c7da1c8c7f5b8ef93e421be4b28a7a27405dccb199798d0d9882cf90831" + labels: + app.kubernetes.io/name: datacatalog + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + initContainers: + - command: + - datacatalog + - --config + - /etc/datacatalog/config/*.yaml + - migrate + - run + image: "ghcr.io/flyteorg/datacatalog:v0.3.4" + imagePullPolicy: "IfNotPresent" + name: run-migrations + volumeMounts: + + - mountPath: /etc/datacatalog/config + name: config-volume + containers: + - command: + - datacatalog + - --config + - /etc/datacatalog/config/*.yaml + - serve + image: "ghcr.io/flyteorg/datacatalog:v0.3.4" + imagePullPolicy: "IfNotPresent" + name: datacatalog + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: 500m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + volumeMounts: + + - mountPath: /etc/datacatalog/config + name: config-volume + serviceAccountName: datacatalog + volumes: + + - emptyDir: {} + name: shared-data + - configMap: + name: datacatalog-config + name: config-volume +--- +# Source: flyte/templates/minio/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: default + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte + template: + metadata: + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "minio/minio:RELEASE.2020-12-16T05-05-17Z" + imagePullPolicy: "IfNotPresent" + name: minio + args: + - server + - /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: miniostorage + ports: + - containerPort: 9000 + name: minio + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - name: minio-storage + mountPath: /var/lib/minioql/data + volumes: + - name: minio-storage + emptyDir: {} +--- +# Source: flyte/templates/postgres/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: default + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte + template: + metadata: + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: "postgres:10.16-alpine" + imagePullPolicy: "IfNotPresent" + name: postgres + env: + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + ports: + - containerPort: 5432 + name: postgres + resources: + limits: + cpu: 1000m + memory: 512Mi + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - name: postgres-storage + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-storage + emptyDir: {} +--- +# Source: flyte/templates/propeller/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: default + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "911c7db31403549038457a5557608a310f4353ccf6e3fa5d0563e81740f11c1" + labels: + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm + spec: + containers: + - command: + - flytepropeller + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: "ghcr.io/flyteorg/flytepropeller:v0.10.10" + imagePullPolicy: "IfNotPresent" + name: flytepropeller + ports: + - containerPort: 10254 + resources: + limits: + cpu: 200m + ephemeral-storage: 100Mi + memory: 200Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: auth + mountPath: /etc/secrets/ + serviceAccountName: flytepropeller + volumes: + - configMap: + name: flyte-propeller-config + name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth +--- +# Source: flyte/templates/propeller/webhook.yaml +# Create the actual deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-pod-webhook + namespace: default + labels: + app: flyte-pod-webhook +spec: + selector: + matchLabels: + app: flyte-pod-webhook + template: + metadata: + labels: + app: flyte-pod-webhook + app.kubernetes.io/name: flyte-pod-webhook + app.kubernetes.io/version: v0.10.10 + annotations: + configChecksum: "911c7db31403549038457a5557608a310f4353ccf6e3fa5d0563e81740f11c1" + spec: + serviceAccountName: flyte-pod-webhook + initContainers: + - name: generate-secrets + image: "ghcr.io/flyteorg/flytepropeller:v0.10.10" + imagePullPolicy: "IfNotPresent" + command: + - flytepropeller + args: + - webhook + - init-certs + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + containers: + - name: webhook + image: "ghcr.io/flyteorg/flytepropeller:v0.10.10" + imagePullPolicy: "IfNotPresent" + command: + - flytepropeller + args: + - webhook + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + readOnly: true + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: config-volume + configMap: + name: flyte-propeller-config + - name: webhook-certs + secret: + secretName: flyte-pod-webhook +--- +# Source: flyte/templates/admin/cronjob.yaml +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: syncresources + namespace: default + labels: + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-0.2.0 + app.kubernetes.io/managed-by: Helm +spec: + schedule: '*/1 * * * *' + jobTemplate: + spec: + template: + spec: + containers: + - command: + - flyteadmin + - --config + - /etc/flyte/config/*.yaml + - clusterresource + - sync + image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + imagePullPolicy: "IfNotPresent" + name: sync-cluster-resources + volumeMounts: + + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: OnFailure + serviceAccountName: flyteadmin + volumes: + + - configMap: + name: clusterresource-template + name: resource-templates + - configMap: + name: flyte-admin-config + name: config-volume +--- +# Source: flyte/templates/common/ingress.yaml +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: flyte + namespace: default +spec: + rules: + - http: + paths: + # This is useful only for frontend development + - path: /__webpack_hmr + pathType: ImplementationSpecific + backend: + serviceName: flyteconsole + servicePort: 80 + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 87 + # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml + - path: /console + pathType: ImplementationSpecific + backend: + serviceName: flyteconsole + servicePort: 80 + - path: /console/* + pathType: ImplementationSpecific + backend: + serviceName: flyteconsole + servicePort: 80 + - path: /api + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /api/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /healthcheck + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /v1/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /.well-known + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /.well-known/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /login + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /login/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /logout + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /logout/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /callback + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /callback/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /me + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /config + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + - path: /config/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 80 + # NOTE: Port 81 in flyteadmin is the GRPC server port for FlyteAdmin. + - path: /flyteidl.service.AdminService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AdminService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 + - path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific + backend: + serviceName: flyteadmin + servicePort: 81 +--- +# Source: flyte/charts/kubernetes-dashboard/templates/clusterrole-readonly.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Source: flyte/charts/kubernetes-dashboard/templates/clusterrolebinding-readonly.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Source: flyte/charts/kubernetes-dashboard/templates/ingress.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Source: flyte/charts/kubernetes-dashboard/templates/networkpolicy.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Source: flyte/charts/kubernetes-dashboard/templates/pdb.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Source: flyte/charts/kubernetes-dashboard/templates/psp.yaml +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# Source: flyte/charts/contour/templates/certgen/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-contour-contour-certgen + namespace: default + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen +--- +# Source: flyte/charts/contour/templates/certgen/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: flyte-contour-contour-certgen + namespace: default + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update +--- +# Source: flyte/charts/contour/templates/certgen/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: flyte-contour-contour-certgen + namespace: default + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: flyte-contour-contour-certgen +subjects: + - kind: ServiceAccount + name: flyte-contour-contour-certgen +--- +# Source: flyte/charts/contour/templates/certgen/job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: flyte-contour-contour-certgen + namespace: default + annotations: + "helm.sh/hook": "pre-install,pre-upgrade" + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen +spec: + ttlSecondsAfterFinished: 0 + template: + metadata: + labels: + app.kubernetes.io/name: contour + helm.sh/chart: contour-4.1.2 + app.kubernetes.io/instance: flyte + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: contour-certgen + spec: + + containers: + - name: contour + image: docker.io/bitnami/contour:1.12.0-debian-10-r0 + imagePullPolicy: IfNotPresent + command: + - contour + args: + - certgen + - --kube + - --incluster + - --overwrite + - --secrets-format=compact + - --namespace=$(CONTOUR_NAMESPACE) + env: + - name: CONTOUR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + restartPolicy: Never + serviceAccountName: flyte-contour-contour-certgen + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: + runAsNonRoot: true + parallelism: 1 + completions: 1 + backoffLimit: 1 diff --git a/helm/values-gcp.yaml b/helm/values-gcp.yaml index 8f9a0069f8..7fa40b2a3f 100644 --- a/helm/values-gcp.yaml +++ b/helm/values-gcp.yaml @@ -4,7 +4,7 @@ common: databaseSecret: {} - ingress: + ingress: {} # ----------------------------------------------------- # Core dependencies that should be configured for Flyte to work on any platform @@ -45,4 +45,4 @@ configmap: sidecar: sidecar spark: spark container_array: k8s-array - pytorch: pytorch \ No newline at end of file + pytorch: pytorch diff --git a/script/generate_helm.sh b/script/generate_helm.sh new file mode 100644 index 0000000000..047db9a318 --- /dev/null +++ b/script/generate_helm.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -ex + +echo "Installing Helm" +# All the values files to be built +DEPLOYMENT=${1:-sandbox eks gcp} + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +for deployment in ${DEPLOYMENT}; do + helm template flyte ${DIR}/../helm/ -f ${DIR}/../helm/values-${deployment}.yaml > ${DIR}/../deployment/${deployment}/flyte_helm_generated.yaml +done + +# This section is used by GitHub workflow to ensure that the generation step was run +if [ -n "$DELTA_CHECK" ]; then + DIRTY=$(git status --porcelain) + if [ -n "$DIRTY" ]; then + echo "FAILED: helm code updated without commiting generated code." + echo "Ensure make helm has run and all changes are committed." + DIFF=$(git diff) + echo "diff detected: $DIFF" + DIFF=$(git diff --name-only) + echo "files different: $DIFF" + exit 1 + else + echo "SUCCESS: Generated code is up to date." + fi +fi From 848845a522bcd3bd05a4aadc01efec8ff967ed4c Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Mon, 24 May 2021 17:29:00 -0700 Subject: [PATCH 84/92] add --dependency-update to helm command Signed-off-by: Haytham Abuelfutuh --- script/generate_helm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate_helm.sh b/script/generate_helm.sh index 047db9a318..f067999db4 100644 --- a/script/generate_helm.sh +++ b/script/generate_helm.sh @@ -9,7 +9,7 @@ DEPLOYMENT=${1:-sandbox eks gcp} DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" for deployment in ${DEPLOYMENT}; do - helm template flyte ${DIR}/../helm/ -f ${DIR}/../helm/values-${deployment}.yaml > ${DIR}/../deployment/${deployment}/flyte_helm_generated.yaml + helm template flyte ${DIR}/../helm/ -f ${DIR}/../helm/values-${deployment}.yaml --dependency-update > ${DIR}/../deployment/${deployment}/flyte_helm_generated.yaml done # This section is used by GitHub workflow to ensure that the generation step was run From b060572d7511d36f4fd2417df01f9a176d05a508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Brunk?= Date: Tue, 25 May 2021 14:11:01 +0200 Subject: [PATCH 85/92] Don't hardcode namespace in flyte-admin-auth secret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sören Brunk Signed-off-by: Haytham Abuelfutuh --- helm/templates/admin/deployment.yaml | 7 ------- helm/templates/admin/secret-auth.yaml | 6 ++++++ helm/templates/propeller/deployment.yaml | 9 --------- helm/templates/propeller/secret-auth.yaml | 8 ++++++++ helm/values.yaml | 2 +- 5 files changed, 15 insertions(+), 17 deletions(-) create mode 100644 helm/templates/admin/secret-auth.yaml create mode 100644 helm/templates/propeller/secret-auth.yaml diff --git a/helm/templates/admin/deployment.yaml b/helm/templates/admin/deployment.yaml index 8d586b41ee..8650e23b75 100644 --- a/helm/templates/admin/deployment.yaml +++ b/helm/templates/admin/deployment.yaml @@ -1,10 +1,3 @@ -apiVersion: v1 -kind: Secret -metadata: - name: flyte-admin-auth - namespace: flyte -type: Opaque ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/helm/templates/admin/secret-auth.yaml b/helm/templates/admin/secret-auth.yaml new file mode 100644 index 0000000000..0d5bf72fca --- /dev/null +++ b/helm/templates/admin/secret-auth.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: {{ template "flyte.namespace" . }} +type: Opaque diff --git a/helm/templates/propeller/deployment.yaml b/helm/templates/propeller/deployment.yaml index ce9d6cc805..7b5ae7e63a 100644 --- a/helm/templates/propeller/deployment.yaml +++ b/helm/templates/propeller/deployment.yaml @@ -1,12 +1,3 @@ -apiVersion: v1 -kind: Secret -metadata: - name: flyte-propeller-auth - namespace: {{ template "flyte.namespace" . }} -type: Opaque -stringData: - client_secret: foobar ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/helm/templates/propeller/secret-auth.yaml b/helm/templates/propeller/secret-auth.yaml new file mode 100644 index 0000000000..0430727057 --- /dev/null +++ b/helm/templates/propeller/secret-auth.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: {{ template "flyte.namespace" . }} +type: Opaque +stringData: + client_secret: foobar diff --git a/helm/values.yaml b/helm/values.yaml index e0c9d51b62..4fe7ac24f4 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -16,7 +16,7 @@ flyteadmin: image: # -- Docker image for Flyteadmin deployment repository: ghcr.io/flyteorg/flyteadmin - tag: v0.5.4 + tag: v0.5.7 pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteadmin deployment resources: From c7059787165486cb3a350b87b186798f0a3f9605 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 25 May 2021 07:29:23 -0700 Subject: [PATCH 86/92] generate helm Signed-off-by: Haytham Abuelfutuh --- deployment/eks/flyte_helm_generated.yaml | 18 +++++++++--------- deployment/gcp/flyte_helm_generated.yaml | 18 +++++++++--------- deployment/sandbox/flyte_helm_generated.yaml | 18 +++++++++--------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index bde28c9755..fd48e1af9c 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -50,12 +50,12 @@ metadata: helm.sh/chart: flyte-0.2.0 app.kubernetes.io/managed-by: Helm --- -# Source: flyte/templates/admin/deployment.yaml +# Source: flyte/templates/admin/secret-auth.yaml apiVersion: v1 kind: Secret metadata: name: flyte-admin-auth - namespace: flyte + namespace: default type: Opaque --- # Source: flyte/templates/common/secret.yaml @@ -67,7 +67,7 @@ stringData: pass.txt: awesomesauce type: Opaque --- -# Source: flyte/templates/propeller/deployment.yaml +# Source: flyte/templates/propeller/secret-auth.yaml apiVersion: v1 kind: Secret metadata: @@ -1292,7 +1292,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1309,7 +1309,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: seed-projects volumeMounts: @@ -1323,7 +1323,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1334,7 +1334,7 @@ spec: - mountPath: /etc/flyte/config name: config-volume - name: generate-secrets - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -1355,7 +1355,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1868,7 +1868,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index 3bc0f3ab7c..ad867a1fd3 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -112,15 +112,15 @@ metadata: helm.sh/chart: flyte-0.2.0 app.kubernetes.io/managed-by: Helm --- -# Source: flyte/templates/admin/deployment.yaml +# Source: flyte/templates/admin/secret-auth.yaml apiVersion: v1 kind: Secret metadata: name: flyte-admin-auth - namespace: flyte + namespace: default type: Opaque --- -# Source: flyte/templates/propeller/deployment.yaml +# Source: flyte/templates/propeller/secret-auth.yaml apiVersion: v1 kind: Secret metadata: @@ -3137,7 +3137,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -3153,7 +3153,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: seed-projects volumeMounts: @@ -3166,7 +3166,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -3176,7 +3176,7 @@ spec: - mountPath: /etc/flyte/config name: config-volume - name: generate-secrets - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -3197,7 +3197,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -3778,7 +3778,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index 25432e99c2..34b37f1aae 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -176,15 +176,15 @@ metadata: name: kubernetes-dashboard-key-holder type: Opaque --- -# Source: flyte/templates/admin/deployment.yaml +# Source: flyte/templates/admin/secret-auth.yaml apiVersion: v1 kind: Secret metadata: name: flyte-admin-auth - namespace: flyte + namespace: default type: Opaque --- -# Source: flyte/templates/propeller/deployment.yaml +# Source: flyte/templates/propeller/secret-auth.yaml apiVersion: v1 kind: Secret metadata: @@ -3301,7 +3301,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -3317,7 +3317,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: seed-projects volumeMounts: @@ -3330,7 +3330,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -3340,7 +3340,7 @@ spec: - mountPath: /etc/flyte/config name: config-volume - name: generate-secrets - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -3361,7 +3361,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -3816,7 +3816,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "ghcr.io/flyteorg/flyteadmin:v0.5.4" + image: "ghcr.io/flyteorg/flyteadmin:v0.5.7" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: From 32f27861bbd2065761039af83bf7b913771a0820 Mon Sep 17 00:00:00 2001 From: Ketan Umare <16888709+kumare3@users.noreply.github.com> Date: Thu, 27 May 2021 09:43:56 -0700 Subject: [PATCH 87/92] Helm template upgrade to auto-generate co-pilot configuration (#1056) * Helm template upgrade to auto-generate co-pilot configuration Signed-off-by: Ketan Umare * using common storage base for all storage and co-pilot Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- deployment/eks/flyte_helm_generated.yaml | 32 +++++++++---------- deployment/gcp/flyte_helm_generated.yaml | 21 ++++++------ deployment/sandbox/flyte_helm_generated.yaml | 10 +++--- helm/templates/_helpers.tpl | 19 +++++++++-- .../admin/cluster_resource_configmap.yaml | 1 + helm/values-eks.yaml | 18 ----------- helm/values.yaml | 23 ------------- 7 files changed, 49 insertions(+), 75 deletions(-) diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index fd48e1af9c..4ac91f10a5 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -127,22 +127,6 @@ data: limits.cpu: {{ projectQuotaCpu }} limits.memory: {{ projectQuotaMemory }} - ac_project_copilot_dataconfig.yaml: | - kind: ConfigMap - apiVersion: v1 - metadata: - name: flyte-data-config - namespace: {{ namespace }} - data: - config.yaml: | - storage: - type: s3 - container: - connection: - auth-type: iam - region: - enable-multicontainer: true - ad_spark_role.yaml: | apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role @@ -192,6 +176,22 @@ data: - kind: ServiceAccount name: spark namespace: {{ namespace }} + + zz_copilot_config.yaml: | + kind: ConfigMap + apiVersion: v1 + metadata: + name: flyte-data-config + namespace: {{ namespace }} + data: + config.yaml: | + storage: + type: s3 + container: "" + connection: + auth-type: iam + region: + enable-multicontainer: true --- # Source: flyte/templates/admin/configmap.yaml apiVersion: v1 diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index ad867a1fd3..10d17c80ec 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -192,24 +192,23 @@ data: limits.cpu: {{ projectQuotaCpu }} limits.memory: {{ projectQuotaMemory }} - ac_project_copilot_dataconfig.yaml: | + zz_copilot_config.yaml: | kind: ConfigMap apiVersion: v1 metadata: name: flyte-data-config namespace: {{ namespace }} data: - config.yaml: | + config.yaml: | storage: - connection: - access-key: minio - auth-type: accesskey - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - secret-key: miniostorage - type: minio - container: my-s3-bucket + type: stow + stow: + kind: google + config: + json: "" + project_id: + scopes: https://www.googleapis.com/auth/devstorage.read_write + container: "" enable-multicontainer: true --- # Source: flyte/templates/admin/configmap.yaml diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index 34b37f1aae..01a082e2cc 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -283,24 +283,24 @@ data: limits.cpu: {{ projectQuotaCpu }} limits.memory: {{ projectQuotaMemory }} - ac_project_copilot_dataconfig.yaml: | + zz_copilot_config.yaml: | kind: ConfigMap apiVersion: v1 metadata: name: flyte-data-config namespace: {{ namespace }} data: - config.yaml: | + config.yaml: | storage: + type: minio + container: "my-s3-bucket" connection: access-key: minio auth-type: accesskey + secret-key: miniostorage disable-ssl: true endpoint: http://minio.flyte.svc.cluster.local:9000 region: us-east-1 - secret-key: miniostorage - type: minio - container: my-s3-bucket enable-multicontainer: true --- # Source: flyte/templates/admin/configmap.yaml diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 153591e02f..0229d30271 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -164,7 +164,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} {{- end }} -{{- define "storage" -}} +{{- define "storage.base" -}} storage: {{- if eq .Values.storage.type "s3" }} type: s3 @@ -196,6 +196,21 @@ storage: {{ toYaml . | nindent 2 }} {{- end }} {{- end }} +{{- end }} + +{{- define "storage" -}} +{{ include "storage.base" .}} limits: maxDownloadMBs: 10 -{{- end }} \ No newline at end of file +{{- end }} + +{{- define "copilot.config" -}} +kind: ConfigMap +apiVersion: v1 +metadata: + name: flyte-data-config + namespace: {{`{{ namespace }}`}} +data: + config.yaml: | {{ include "storage.base" . | nindent 4 }} + enable-multicontainer: true +{{- end }} diff --git a/helm/templates/admin/cluster_resource_configmap.yaml b/helm/templates/admin/cluster_resource_configmap.yaml index f3b7eeae1c..f5f62f58b5 100644 --- a/helm/templates/admin/cluster_resource_configmap.yaml +++ b/helm/templates/admin/cluster_resource_configmap.yaml @@ -9,4 +9,5 @@ data: {{- range .Values.cluster_resource_manager.templates }} {{ .key }}.yaml: | {{ .value | nindent 4 }} {{- end }} + zz_copilot_config.yaml: | {{ include "copilot.config" . | nindent 4 }} {{- end }} diff --git a/helm/values-eks.yaml b/helm/values-eks.yaml index 80bfc94513..a244a4a634 100644 --- a/helm/values-eks.yaml +++ b/helm/values-eks.yaml @@ -400,24 +400,6 @@ cluster_resource_manager: limits.cpu: {{ projectQuotaCpu }} limits.memory: {{ projectQuotaMemory }} - # -- Template copilot configuration - - key: ac_project_copilot_dataconfig - value: | - kind: ConfigMap - apiVersion: v1 - metadata: - name: flyte-data-config - namespace: {{ namespace }} - data: - config.yaml: | - storage: - type: s3 - container: - connection: - auth-type: iam - region: - enable-multicontainer: true - # -- Template for spark role - key: ad_spark_role value: | diff --git a/helm/values.yaml b/helm/values.yaml index 4fe7ac24f4..dc30311cac 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -739,29 +739,6 @@ cluster_resource_manager: limits.cpu: {{ projectQuotaCpu }} limits.memory: {{ projectQuotaMemory }} - # -- Template for spark role - - key: ac_project_copilot_dataconfig - value: | - kind: ConfigMap - apiVersion: v1 - metadata: - name: flyte-data-config - namespace: {{ namespace }} - data: - config.yaml: | - storage: - connection: - access-key: minio - auth-type: accesskey - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - secret-key: miniostorage - type: minio - container: my-s3-bucket - enable-multicontainer: true - - # -------------------------------------------------------- # Optional Plugins # -------------------------------------------------------- From b637c2171c2e6f2207c6c162e8b7198414168421 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Fri, 28 May 2021 09:41:21 -0700 Subject: [PATCH 88/92] Install deps once Signed-off-by: Haytham Abuelfutuh --- script/generate_helm.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/script/generate_helm.sh b/script/generate_helm.sh index f067999db4..5507b86346 100644 --- a/script/generate_helm.sh +++ b/script/generate_helm.sh @@ -2,16 +2,21 @@ set -ex -echo "Installing Helm" +echo "Generating Helm" # All the values files to be built DEPLOYMENT=${1:-sandbox eks gcp} DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +helm dep update ${DIR}/../helm/ + for deployment in ${DEPLOYMENT}; do - helm template flyte ${DIR}/../helm/ -f ${DIR}/../helm/values-${deployment}.yaml --dependency-update > ${DIR}/../deployment/${deployment}/flyte_helm_generated.yaml + helm template flyte ${DIR}/../helm/ -f ${DIR}/../helm/values-${deployment}.yaml > ${DIR}/../deployment/${deployment}/flyte_helm_generated.yaml done +echo "Generating helm docs" +helm-docs -t ${DIR}/../helm/README.md.gotmpl ${DIR}/../helm/ + # This section is used by GitHub workflow to ensure that the generation step was run if [ -n "$DELTA_CHECK" ]; then DIRTY=$(git status --porcelain) From fba8387c04c58e6a31875ef4ccefa18d58304abc Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Fri, 28 May 2021 10:18:35 -0700 Subject: [PATCH 89/92] Install helm-docs if not found Signed-off-by: Haytham Abuelfutuh --- helm/README.md | 46 +++++++++++++++++++++++++++-------------- script/generate_helm.sh | 5 +++++ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/helm/README.md b/helm/README.md index ba32673ebc..76f70a0cd3 100644 --- a/helm/README.md +++ b/helm/README.md @@ -51,13 +51,12 @@ helm upgrade -f values-sandbox.yaml flyte . | Key | Type | Default | Description | |-----|------|---------|-------------| -| cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"},{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | -| cluster_resource_manager.config | object | `{"cluster_resources":{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}}` | Configmap for ClusterResource parameters | -| cluster_resource_manager.config.cluster_resources | object | `{"customData":{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}],"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}],"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. | +| cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | +| cluster_resource_manager.config | object | `{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}}` | Configmap for ClusterResource parameters | +| cluster_resource_manager.config.cluster_resources | object | `{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. | | cluster_resource_manager.enabled | bool | `true` | Enables the Cluster resource manager component | -| cluster_resource_manager.templates | list | `[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"},{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}]` | Resource templates that should be applied | +| cluster_resource_manager.templates | list | `[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]` | Resource templates that should be applied | | cluster_resource_manager.templates[0] | object | `{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"}` | Template for namespaces resources | -| cluster_resource_manager.templates[2] | object | `{"key":"ac_project_copilot_dataconfig","value":"kind: ConfigMap\napiVersion: v1\nmetadata:\n name: flyte-data-config\n namespace: {{ namespace }}\ndata:\n config.yaml: |\n storage:\n connection:\n access-key: minio\n auth-type: accesskey\n disable-ssl: true\n endpoint: http://minio.flyte.svc.cluster.local:9000\n region: us-east-1\n secret-key: miniostorage\n type: minio\n container: my-s3-bucket\n enable-multicontainer: true\n"}` | Template for spark role | | common.databaseSecret.name | string | `""` | Specify name of K8s Secret which contains Database password. Leave it empty if you don't need this Secret | | common.databaseSecret.secretManifest | object | `{}` | Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets | | common.flyteNamespaceTemplate.enabled | bool | `false` | | @@ -66,14 +65,18 @@ helm upgrade -f values-sandbox.yaml flyte . | common.ingress.enabled | bool | `true` | | | common.ingress.separateGrpcIngress | bool | `false` | | | common.ingress.separateGrpcIngressAnnotations."nginx.ingress.kubernetes.io/backend-protocol" | string | `"GRPC"` | | -| common.ingress.tls.enabled | bool | `true` | | +| common.ingress.tls.enabled | bool | `false` | | | common.ingress.webpackHMR | bool | `false` | | -| configmap.admin | object | `{"admin":{"endpoint":"flyteadmin:81","insecure":true},"event":{"capacity":1000,"rate":500,"type":"admin"}}` | Admin Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/subworkflow/launchplan#AdminConfig) | +| configmap.admin | object | `{"admin":{"clientId":"flytepropeller","clientSecretLocation":"/etc/secrets/client_secret","endpoint":"flyteadmin:81","insecure":true},"event":{"capacity":1000,"rate":500,"type":"admin"}}` | Admin Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/subworkflow/launchplan#AdminConfig) | +| configmap.adminServer | object | `{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"https://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":1,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | +| configmap.adminServer.auth | object | `{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"https://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}}` | Authentication configuration | +| configmap.adminServer.server.security.secure | bool | `false` | Controls whether to serve requests over SSL/TLS. | +| configmap.adminServer.server.security.useAuth | bool | `false` | Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. | | configmap.catalog | object | `{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}}` | Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) | | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config","DISABLE_AUTH":"1"}` | Configuration for Flyte console UI | | configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | | configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | -| configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}}` | Core propeller configuration | +| configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | | configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | | configmap.domain | object | `{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]}` | Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. | @@ -83,9 +86,11 @@ helm upgrade -f values-sandbox.yaml flyte . | configmap.k8s | object | `{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}}}` | Kubernetes specific Flyte configuration | | configmap.k8s.plugins.k8s | object | `{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}` | Configuration section for all K8s specific plugins [Configuration structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/flytek8s/config) | | configmap.logger | object | `{"logger":{"level":4,"show-source":true}}` | Logger configuration | +| configmap.remoteData.remoteData.region | string | `"us-east-1"` | | +| configmap.remoteData.remoteData.scheme | string | `"local"` | | +| configmap.remoteData.remoteData.signedUrls.durationMinutes | int | `3` | | | configmap.resource_manager | object | `{"propeller":{"resourcemanager":{"redis":{"hostKey":"mypassword","hostPath":"redis-resource-manager:6379"},"resourceMaxQuota":10000,"type":"redis"}}}` | Resource manager configuration | | configmap.resource_manager.propeller | object | `{"resourcemanager":{"redis":{"hostKey":"mypassword","hostPath":"redis-resource-manager:6379"},"resourceMaxQuota":10000,"type":"redis"}}` | resource manager configuration | -| configmap.server | object | `{"flyteadmin":{"eventVersion":1,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | | configmap.task_logs | object | `{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":false}}}` | Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various logging engines | | configmap.task_logs.plugins.logs.cloudwatch-enabled | bool | `false` | One option is to enable cloudwatch logging for EKS, update the region and log group accordingly | | configmap.task_resource_defaults | object | `{"task_resources":{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"8Gi","storage":"20Mi"}}}` | Task default resources configuration Refer to the full [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#TaskResourceConfiguration). | @@ -103,35 +108,37 @@ helm upgrade -f values-sandbox.yaml flyte . | datacatalog.configPath | string | `"/etc/datacatalog/config/*.yaml"` | Default regex string for searching configuration files | | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | | | datacatalog.image.repository | string | `"ghcr.io/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| datacatalog.image.tag | string | `"v0.3.2"` | | +| datacatalog.image.tag | string | `"v0.3.4"` | | | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | | datacatalog.resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Datacatalog deployment | | datacatalog.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"}` | Service settings for Datacatalog | -| datacatalog.serviceAccount | object | `{"annotations":{},"create":true}` | Configuration for service accounts for Datacatalog | +| datacatalog.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":{}}` | Configuration for service accounts for Datacatalog | | datacatalog.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Datacatalog pods | | datacatalog.serviceAccount.create | bool | `true` | Should a service account be created for Datacatalog | +| datacatalog.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | | datacatalog.tolerations | list | `[]` | tolerations for Datacatalog deployment | | db.database | object | `{}` | | | flyteadmin.affinity | object | `{}` | affinity for Flyteadmin deployment | | flyteadmin.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | | | flyteadmin.image.repository | string | `"ghcr.io/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyteadmin.image.tag | string | `"v0.4.14"` | | +| flyteadmin.image.tag | string | `"v0.5.7"` | | | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | | flyteadmin.replicaCount | int | `1` | Replicas count for Flyteadmin deployment | | flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment | | flyteadmin.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin | -| flyteadmin.serviceAccount | object | `{"annotations":{},"create":true}` | Configuration for service accounts for FlyteAdmin | +| flyteadmin.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":{}}` | Configuration for service accounts for FlyteAdmin | | flyteadmin.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Flyteadmin pods | | flyteadmin.serviceAccount.create | bool | `true` | Should a service account be created for flyteadmin | +| flyteadmin.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment | | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | | | flyteconsole.image.repository | string | `"ghcr.io/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyteconsole.image.tag | string | `"v0.19.7"` | | +| flyteconsole.image.tag | string | `"v0.20.0"` | | | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | | flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment | @@ -143,14 +150,15 @@ helm upgrade -f values-sandbox.yaml flyte . | flytepropeller.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | | | flytepropeller.image.repository | string | `"ghcr.io/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flytepropeller.image.tag | string | `"v0.10.4"` | | +| flytepropeller.image.tag | string | `"v0.10.10"` | | | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | | flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytepropeller deployment | -| flytepropeller.serviceAccount | object | `{"annotations":{},"create":true}` | Configuration for service accounts for FlytePropeller | +| flytepropeller.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":{}}` | Configuration for service accounts for FlytePropeller | | flytepropeller.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to FlytePropeller pods | | flytepropeller.serviceAccount.create | bool | `true` | Should a service account be created for FlytePropeller | +| flytepropeller.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | | flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment | | kubernetes-dashboard.enabled | bool | `false` | | | minio.affinity | object | `{}` | affinity for Minio deployment | @@ -213,6 +221,12 @@ helm upgrade -f values-sandbox.yaml flyte . | storage.s3 | object | `{"region":"us-east-1"}` | settings for storage type s3 | | storage.type | string | `"sandbox"` | Sets the storage type. Supported values are sandbox, s3, gcs and custom. | | tf_operator.enabled | bool | `false` | | +| webhook.enabled | bool | `true` | enable or disable secrets webhook | +| webhook.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"}` | Service settings for the webhook | +| webhook.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":{}}` | Configuration for service accounts for the webhook | +| webhook.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to the webhook | +| webhook.serviceAccount.create | bool | `true` | Should a service account be created for the webhook | +| webhook.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | | workflow_notifications | object | `{"config":{},"enabled":false}` | **Optional Component** Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of various events in their workflows | | workflow_scheduler.config | object | `{}` | | | workflow_scheduler.enabled | bool | `false` | | diff --git a/script/generate_helm.sh b/script/generate_helm.sh index 5507b86346..bcd9626c86 100644 --- a/script/generate_helm.sh +++ b/script/generate_helm.sh @@ -15,6 +15,11 @@ for deployment in ${DEPLOYMENT}; do done echo "Generating helm docs" +if ! command -v helm-docs &> /dev/null +then + GO111MODULE=on go get github.com/norwoodj/helm-docs/cmd/helm-docs +fi + helm-docs -t ${DIR}/../helm/README.md.gotmpl ${DIR}/../helm/ # This section is used by GitHub workflow to ensure that the generation step was run From 833f1e1eb874c1ca2af8996af22655a5ae8ca333 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Fri, 28 May 2021 14:58:05 -0700 Subject: [PATCH 90/92] bump Signed-off-by: Haytham Abuelfutuh From 6399932253c38541081dcff857f6f688a7637a81 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Fri, 28 May 2021 15:03:17 -0700 Subject: [PATCH 91/92] Fix merge Signed-off-by: Haytham Abuelfutuh --- Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile b/Makefile index cf1cd6b112..a5449e4b92 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,6 @@ stats: .PHONY: prepare_artifacts prepare_artifacts: -<<<<<<< HEAD bash script/prepare_artifacts.sh .PHONY: helm_install @@ -64,6 +63,3 @@ helm_install: .PHONY: helm_upgrade helm_upgrade: helm upgrade flyte --debug ./helm -f helm/values-sandbox.yaml --create-namespace -======= - bash script/prepare_artifacts.sh ->>>>>>> master From ff0f087488911d314f613079e276115d034787a6 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Fri, 28 May 2021 17:24:54 -0700 Subject: [PATCH 92/92] Update helm docs Signed-off-by: Haytham Abuelfutuh --- script/generate_helm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/generate_helm.sh b/script/generate_helm.sh index bcd9626c86..f772801e5a 100644 --- a/script/generate_helm.sh +++ b/script/generate_helm.sh @@ -20,7 +20,7 @@ then GO111MODULE=on go get github.com/norwoodj/helm-docs/cmd/helm-docs fi -helm-docs -t ${DIR}/../helm/README.md.gotmpl ${DIR}/../helm/ +~/go/bin/helm-docs -t ${DIR}/../helm/README.md.gotmpl ${DIR}/../helm/ # This section is used by GitHub workflow to ensure that the generation step was run if [ -n "$DELTA_CHECK" ]; then