From 1cfce063b3fdf2cfe830eca3b817c729f37af292 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 1 Jun 2020 21:34:45 -0700 Subject: [PATCH] Milestone v0.4.0 (#331) Versions: IDL: [v0.17.32](https://github.com/lyft/flyteidl/releases/tag/v0.17.30) Admin: [v0.2.8](https://github.com/lyft/flyteadmin/releases/tag/v0.2.8) Propeller: [v0.2.45](https://github.com/lyft/flytepropeller/releases/tag/v0.2.45) DataCatalog: [v0.2.1](https://github.com/lyft/datacatalog/releases/tag/v0.2.1) Flytekit: [v0.8.1](https://github.com/lyft/flytekit/releases/tag/v0.8.1) FlyteConsole: [v0.5.3](https://github.com/lyft/flyteconsole/releases/tag/v0.5.3) FlytePlugins: [v0.3.28](https://github.com/lyft/flyteplugins/releases/tag/v0.3.28) --- CHANGELOG/CHANGELOG-v0.4.0.md | 18 +++++++++++++++++ Makefile | 4 ++-- README.md | 14 ++++++++++--- deployment/eks/flyte_generated.yaml | 20 +++++++++---------- deployment/sandbox/flyte_generated.yaml | 20 +++++++++---------- deployment/test/flyte_generated.yaml | 16 +++++++-------- end2end/tests/endtoend.yaml | 2 +- .../base/admindeployment/deployment.yaml | 6 +++--- kustomize/base/console/deployment.yaml | 4 ++-- kustomize/base/propeller/deployment.yaml | 4 ++-- .../eks/admindeployment/admindeployment.yaml | 6 +++--- .../overlays/eks/admindeployment/cron.yaml | 2 +- .../admindeployment/admindeployment.yaml | 6 +++--- .../sandbox/admindeployment/cron.yaml | 2 +- rsts/conf.py | 2 +- 15 files changed, 76 insertions(+), 50 deletions(-) create mode 100644 CHANGELOG/CHANGELOG-v0.4.0.md diff --git a/CHANGELOG/CHANGELOG-v0.4.0.md b/CHANGELOG/CHANGELOG-v0.4.0.md new file mode 100644 index 0000000000..07311f6bee --- /dev/null +++ b/CHANGELOG/CHANGELOG-v0.4.0.md @@ -0,0 +1,18 @@ +# Flyte V0.3.0 + +## Changes since v0.3.0 + +### Core Platform +- Single Task Execution support (API only, no flytekit yet) +- Pytorch operator and plugin support to run distributed Pytorch jobs natively in k8s +- Performance improvements +- Stability improvements +- Observability improvements + +### Flytekit (SDK improvements) +- Serialization of flyte workflows +- Registration of flyte entities from files +- Write tasks and subworkflows in dynamic job spec + +### Flyte Console (UI) +- Hide archived workflows in UI diff --git a/Makefile b/Makefile index 4747b76925..b81d64bfc5 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,9 @@ generate-local-docs: # Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir .PHONY: generate-docs generate-docs: generate-dependent-repo-docs - @FLYTEKIT_VERSION=0.7.0 ./script/generate_docs.sh + @FLYTEKIT_VERSION=0.8.1 ./script/generate_docs.sh # updates referenced docs from other repositories (e.g. flyteidl, flytekit) .PHONY: generate-dependent-repo-docs generate-dependent-repo-docs: - @FLYTEKIT_VERSION=0.7.0 FLYTEIDL_VERSION=0.17.27 ./script/update_ref_docs.sh + @FLYTEKIT_VERSION=0.8.1 FLYTEIDL_VERSION=0.17.32 ./script/update_ref_docs.sh diff --git a/README.md b/README.md index 5765caf24a..1f4a4fc42a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyte.svg?style=plastic) ![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyte/latest.svg?style=plastic) ![GitHub milestones Completed](https://img.shields.io/github/milestones/closed/lyft/flyte?style=plastic) -![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/4?style=plastic) +![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/5?style=plastic) ![Twitter Follow](https://img.shields.io/twitter/follow/flyteorg?label=Follow&style=social) [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://docs.google.com/forms/d/e/1FAIpQLSf8bNuyhy7rkm77cOXPHIzCm3ApfL7Tdo7NUs6Ej2NOGQ1PYw/viewform?pli=1) @@ -42,8 +42,9 @@ Resources that would help you get a better understanding of Flyte. # Features - - Used at Scale in production by 500+ users at Lyft with more than 400k workflows a month and more than 20+ million container executions per month + - Used at Scale in production by 500+ users at Lyft with more than *900k* workflow executed a month and more than *30+* million container executions per month - Centralized Inventory of Tasks, Workflows and Executions + - Single Task Execution support - Start executing a task and then convert it to a workflow - gRPC / REST interface to define and executes tasks and workflows - Type safe construction of pipelines, each task has an interface which is characterized by its input and outputs. Thus illegal construction of pipelines fails during declaration rather than at runtime @@ -74,8 +75,10 @@ Resources that would help you get a better understanding of Flyte. - Written in Golang and optimized for performance ## Coming Soon - - Single Task Execution support - Reactive pipelines + - Golang CLI + - Grafana templates (user/system observability) + - Arbitrary flytekit less container support - More integrations # Available Plugins @@ -86,6 +89,11 @@ Resources that would help you get a better understanding of Flyte. - K8s Spark (native pyspark and java/scala) - Qubole Hive - Presto Queries + - Pytorch Operator + +## Coming soon + - Sagemaker + - Flink-K8s # Current Usage - Lyft Rideshare diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index 5d61eb951c..ac24d844ab 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -594,7 +594,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.2.7 + app.kubernetes.io/version: 0.2.8 spec: containers: - command: @@ -603,7 +603,7 @@ spec: - --config - /etc/flyte/config/flyteadmin_config.yaml - serve - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -647,7 +647,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -663,7 +663,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -676,7 +676,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -712,13 +712,13 @@ spec: labels: app: flyteconsole app.kubernetes.io/name: flyteconsole - app.kubernetes.io/version: 0.5.2 + app.kubernetes.io/version: 0.5.3 spec: containers: - envFrom: - configMapRef: name: flyte-console-config - image: docker.io/lyft/flyteconsole:v0.5.2 + image: docker.io/lyft/flyteconsole:v0.5.3 name: flyteconsole ports: - containerPort: 8080 @@ -754,7 +754,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.45 spec: containers: - args: @@ -767,7 +767,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.45 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -874,7 +874,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 15078342a8..ef2091f9ba 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -1024,7 +1024,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.2.7 + app.kubernetes.io/version: 0.2.8 spec: containers: - command: @@ -1033,7 +1033,7 @@ spec: - --config - /etc/flyte/config/flyteadmin_config.yaml - serve - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -1084,7 +1084,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -1100,7 +1100,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -1113,7 +1113,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -1149,13 +1149,13 @@ spec: labels: app: flyteconsole app.kubernetes.io/name: flyteconsole - app.kubernetes.io/version: 0.5.2 + app.kubernetes.io/version: 0.5.3 spec: containers: - envFrom: - configMapRef: name: flyte-console-config - image: docker.io/lyft/flyteconsole:v0.5.2 + image: docker.io/lyft/flyteconsole:v0.5.3 name: flyteconsole ports: - containerPort: 8080 @@ -1191,7 +1191,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.45 spec: containers: - args: @@ -1206,7 +1206,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.45 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1521,7 +1521,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 8d5940efb2..4523acd75c 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -644,7 +644,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.2.7 + app.kubernetes.io/version: 0.2.8 spec: containers: - command: @@ -653,7 +653,7 @@ spec: - --config - /etc/flyte/config/flyteadmin_config.yaml - serve - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -704,7 +704,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -720,7 +720,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -733,7 +733,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -772,7 +772,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.45 spec: containers: - args: @@ -785,7 +785,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.45 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1031,7 +1031,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/end2end/tests/endtoend.yaml b/end2end/tests/endtoend.yaml index 89e0736a2f..43a572b94b 100644 --- a/end2end/tests/endtoend.yaml +++ b/end2end/tests/endtoend.yaml @@ -11,7 +11,7 @@ spec: command: - bash - -c - image: docker.io/lyft/flytetester:v0.1.5 + image: docker.io/lyft/flytetester:v0.1.6 imagePullPolicy: IfNotPresent name: flytetester resources: diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml index 3f47f89628..37b0795c96 100644 --- a/kustomize/base/admindeployment/deployment.yaml +++ b/kustomize/base/admindeployment/deployment.yaml @@ -16,7 +16,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.2.7 + app.kubernetes.io/version: 0.2.8 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -31,7 +31,7 @@ spec: name: flyte-admin-config initContainers: - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] volumeMounts: @@ -39,7 +39,7 @@ spec: mountPath: /etc/flyte/config containers: - name: flyteadmin - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "serve"] ports: diff --git a/kustomize/base/console/deployment.yaml b/kustomize/base/console/deployment.yaml index 2d76467475..bfe1fbfd60 100644 --- a/kustomize/base/console/deployment.yaml +++ b/kustomize/base/console/deployment.yaml @@ -16,14 +16,14 @@ spec: labels: app: flyteconsole app.kubernetes.io/name: flyteconsole - app.kubernetes.io/version: 0.5.2 + app.kubernetes.io/version: 0.5.3 spec: volumes: - name: shared-data emptyDir: {} containers: - name: flyteconsole - image: docker.io/lyft/flyteconsole:v0.5.2 + image: docker.io/lyft/flyteconsole:v0.5.3 # args: [] ports: - containerPort: 8080 diff --git a/kustomize/base/propeller/deployment.yaml b/kustomize/base/propeller/deployment.yaml index eabb4710a9..3245df9482 100644 --- a/kustomize/base/propeller/deployment.yaml +++ b/kustomize/base/propeller/deployment.yaml @@ -15,7 +15,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.36 + app.kubernetes.io/version: 0.2.45 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -31,7 +31,7 @@ spec: name: flyte-plugin-config containers: - name: flytepropeller - image: docker.io/lyft/flytepropeller:v0.2.36 + image: docker.io/lyft/flytepropeller:v0.2.45 command: - flytepropeller args: diff --git a/kustomize/overlays/eks/admindeployment/admindeployment.yaml b/kustomize/overlays/eks/admindeployment/admindeployment.yaml index b3261b9cb1..f75eba90b3 100644 --- a/kustomize/overlays/eks/admindeployment/admindeployment.yaml +++ b/kustomize/overlays/eks/admindeployment/admindeployment.yaml @@ -12,7 +12,7 @@ spec: name: clusterresource-template initContainers: - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] @@ -20,7 +20,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"] @@ -28,7 +28,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/overlays/eks/admindeployment/cron.yaml b/kustomize/overlays/eks/admindeployment/cron.yaml index 6b35cb696b..a427fb8d36 100644 --- a/kustomize/overlays/eks/admindeployment/cron.yaml +++ b/kustomize/overlays/eks/admindeployment/cron.yaml @@ -12,7 +12,7 @@ spec: serviceAccountName: flyteadmin containers: - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml index d426a6f992..fedbb5ffc0 100644 --- a/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml +++ b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml @@ -17,7 +17,7 @@ spec: 'until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done;'] - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] @@ -25,7 +25,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"] @@ -33,7 +33,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/overlays/sandbox/admindeployment/cron.yaml b/kustomize/overlays/sandbox/admindeployment/cron.yaml index 6b35cb696b..a427fb8d36 100644 --- a/kustomize/overlays/sandbox/admindeployment/cron.yaml +++ b/kustomize/overlays/sandbox/admindeployment/cron.yaml @@ -12,7 +12,7 @@ spec: serviceAccountName: flyteadmin containers: - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.7 + image: docker.io/lyft/flyteadmin:v0.2.8 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/rsts/conf.py b/rsts/conf.py index d5c58eccda..37f6dda462 100644 --- a/rsts/conf.py +++ b/rsts/conf.py @@ -26,7 +26,7 @@ # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'0.3.0' +release = u'0.4.0' # -- General configuration ---------------------------------------------------