Milestone v0.4.0 (#331)

Versions: IDL: [v0.17.32](https://github.com/lyft/flyteidl/releases/tag/v0.17.30) Admin: [v0.2.8](https://github.com/lyft/flyteadmin/releases/tag/v0.2.8) Propeller: [v0.2.45](https://github.com/lyft/flytepropeller/releases/tag/v0.2.45) DataCatalog: [v0.2.1](https://github.com/lyft/datacatalog/releases/tag/v0.2.1) Flytekit: [v0.8.1](https://github.com/lyft/flytekit/releases/tag/v0.8.1) FlyteConsole: [v0.5.3](https://github.com/lyft/flyteconsole/releases/tag/v0.5.3) FlytePlugins: [v0.3.28](https://github.com/lyft/flyteplugins/releases/tag/v0.3.28)
flyteorg · Jun 2, 2020 · 1cfce06 · 1cfce06
1 parent 38309a3
commit 1cfce06
Show file tree

Hide file tree

Showing 15 changed files with 76 additions and 50 deletions.
diff --git a/CHANGELOG/CHANGELOG-v0.4.0.md b/CHANGELOG/CHANGELOG-v0.4.0.md
@@ -0,0 +1,18 @@
+# Flyte V0.3.0
+
+## Changes since v0.3.0
+
+### Core Platform
+- Single Task Execution support (API only, no flytekit yet)
+- Pytorch operator and plugin support to run distributed Pytorch jobs natively in k8s
+- Performance improvements
+- Stability improvements
+- Observability improvements
+
+### Flytekit (SDK improvements)
+- Serialization of flyte workflows
+- Registration of flyte entities from files
+- Write tasks and subworkflows in dynamic job spec
+
+### Flyte Console (UI)
+- Hide archived workflows in UI
diff --git a/Makefile b/Makefile
@@ -24,9 +24,9 @@ generate-local-docs:
 # Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir
 .PHONY: generate-docs
 generate-docs: generate-dependent-repo-docs
-	@FLYTEKIT_VERSION=0.7.0 ./script/generate_docs.sh
+	@FLYTEKIT_VERSION=0.8.1 ./script/generate_docs.sh
 
 # updates referenced docs from other repositories (e.g. flyteidl, flytekit)
 .PHONY: generate-dependent-repo-docs
 generate-dependent-repo-docs:
-	@FLYTEKIT_VERSION=0.7.0 FLYTEIDL_VERSION=0.17.27 ./script/update_ref_docs.sh
+	@FLYTEKIT_VERSION=0.8.1 FLYTEIDL_VERSION=0.17.32 ./script/update_ref_docs.sh
diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@
 ![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyte.svg?style=plastic)
 ![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyte/latest.svg?style=plastic)
 ![GitHub milestones Completed](https://img.shields.io/github/milestones/closed/lyft/flyte?style=plastic)
-![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/4?style=plastic)
+![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/5?style=plastic)
 ![Twitter Follow](https://img.shields.io/twitter/follow/flyteorg?label=Follow&style=social)
 [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://docs.google.com/forms/d/e/1FAIpQLSf8bNuyhy7rkm77cOXPHIzCm3ApfL7Tdo7NUs6Ej2NOGQ1PYw/viewform?pli=1)
 
@@ -42,8 +42,9 @@ Resources that would help you get a better understanding of Flyte.
 
 
 # Features
- - Used at Scale in production by 500+ users at Lyft with more than 400k workflows a month and more than 20+ million container executions per month
+ - Used at Scale in production by 500+ users at Lyft with more than *900k* workflow executed a month and more than *30+* million container executions per month
  - Centralized Inventory of Tasks, Workflows and Executions
+ - Single Task Execution support - Start executing a task and then convert it to a workflow
  - gRPC / REST interface to define and executes tasks and workflows
  - Type safe construction of pipelines, each task has an interface which is characterized by its input and outputs. Thus illegal construction of pipelines fails during declaration rather than at
    runtime
@@ -74,8 +75,10 @@ Resources that would help you get a better understanding of Flyte.
  - Written in Golang and optimized for performance
 
 ## Coming Soon
- - Single Task Execution support
  - Reactive pipelines
+ - Golang CLI
+ - Grafana templates (user/system observability)
+ - Arbitrary flytekit less container support
  - More integrations
 
 # Available Plugins
@@ -86,6 +89,11 @@ Resources that would help you get a better understanding of Flyte.
  - K8s Spark (native pyspark and java/scala)
  - Qubole Hive
  - Presto Queries
+ - Pytorch Operator
+
+## Coming soon
+ - Sagemaker
+ - Flink-K8s
 
 # Current Usage 
 - Lyft Rideshare

diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml
@@ -594,7 +594,7 @@ spec:
       labels:
         app: flyteadmin
         app.kubernetes.io/name: flyteadmin
-        app.kubernetes.io/version: 0.2.7
+        app.kubernetes.io/version: 0.2.8
     spec:
       containers:
       - command:
@@ -603,7 +603,7 @@ spec:
         - --config
         - /etc/flyte/config/flyteadmin_config.yaml
         - serve
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -647,7 +647,7 @@ spec:
         - /etc/flyte/config/flyteadmin_config.yaml
         - migrate
         - run
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -663,7 +663,7 @@ spec:
         - flytesnacks
         - flytetester
         - flyteexamples
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -676,7 +676,7 @@ spec:
         - /etc/flyte/config/flyteadmin_config.yaml
         - clusterresource
         - sync
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -712,13 +712,13 @@ spec:
       labels:
         app: flyteconsole
         app.kubernetes.io/name: flyteconsole
-        app.kubernetes.io/version: 0.5.2
+        app.kubernetes.io/version: 0.5.3
     spec:
       containers:
       - envFrom:
         - configMapRef:
             name: flyte-console-config
-        image: docker.io/lyft/flyteconsole:v0.5.2
+        image: docker.io/lyft/flyteconsole:v0.5.3
         name: flyteconsole
         ports:
         - containerPort: 8080
@@ -754,7 +754,7 @@ spec:
       labels:
         app: flytepropeller
         app.kubernetes.io/name: flytepropeller
-        app.kubernetes.io/version: 0.2.36
+        app.kubernetes.io/version: 0.2.45
     spec:
       containers:
       - args:
@@ -767,7 +767,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: docker.io/lyft/flytepropeller:v0.2.36
+        image: docker.io/lyft/flytepropeller:v0.2.45
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -874,7 +874,7 @@ spec:
             - /etc/flyte/config/flyteadmin_config.yaml
             - clusterresource
             - sync
-            image: docker.io/lyft/flyteadmin:v0.2.7
+            image: docker.io/lyft/flyteadmin:v0.2.8
             imagePullPolicy: IfNotPresent
             name: sync-cluster-resources
             volumeMounts:

diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml
@@ -1024,7 +1024,7 @@ spec:
       labels:
         app: flyteadmin
         app.kubernetes.io/name: flyteadmin
-        app.kubernetes.io/version: 0.2.7
+        app.kubernetes.io/version: 0.2.8
     spec:
       containers:
       - command:
@@ -1033,7 +1033,7 @@ spec:
         - --config
         - /etc/flyte/config/flyteadmin_config.yaml
         - serve
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -1084,7 +1084,7 @@ spec:
         - /etc/flyte/config/flyteadmin_config.yaml
         - migrate
         - run
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -1100,7 +1100,7 @@ spec:
         - flytesnacks
         - flytetester
         - flyteexamples
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -1113,7 +1113,7 @@ spec:
         - /etc/flyte/config/flyteadmin_config.yaml
         - clusterresource
         - sync
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -1149,13 +1149,13 @@ spec:
       labels:
         app: flyteconsole
         app.kubernetes.io/name: flyteconsole
-        app.kubernetes.io/version: 0.5.2
+        app.kubernetes.io/version: 0.5.3
     spec:
       containers:
       - envFrom:
         - configMapRef:
             name: flyte-console-config
-        image: docker.io/lyft/flyteconsole:v0.5.2
+        image: docker.io/lyft/flyteconsole:v0.5.3
         name: flyteconsole
         ports:
         - containerPort: 8080
@@ -1191,7 +1191,7 @@ spec:
       labels:
         app: flytepropeller
         app.kubernetes.io/name: flytepropeller
-        app.kubernetes.io/version: 0.2.36
+        app.kubernetes.io/version: 0.2.45
     spec:
       containers:
       - args:
@@ -1206,7 +1206,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: docker.io/lyft/flytepropeller:v0.2.36
+        image: docker.io/lyft/flytepropeller:v0.2.45
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -1521,7 +1521,7 @@ spec:
             - /etc/flyte/config/flyteadmin_config.yaml
             - clusterresource
             - sync
-            image: docker.io/lyft/flyteadmin:v0.2.7
+            image: docker.io/lyft/flyteadmin:v0.2.8
             imagePullPolicy: IfNotPresent
             name: sync-cluster-resources
             volumeMounts:

diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml
@@ -644,7 +644,7 @@ spec:
       labels:
         app: flyteadmin
         app.kubernetes.io/name: flyteadmin
-        app.kubernetes.io/version: 0.2.7
+        app.kubernetes.io/version: 0.2.8
     spec:
       containers:
       - command:
@@ -653,7 +653,7 @@ spec:
         - --config
         - /etc/flyte/config/flyteadmin_config.yaml
         - serve
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -704,7 +704,7 @@ spec:
         - /etc/flyte/config/flyteadmin_config.yaml
         - migrate
         - run
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -720,7 +720,7 @@ spec:
         - flytesnacks
         - flytetester
         - flyteexamples
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -733,7 +733,7 @@ spec:
         - /etc/flyte/config/flyteadmin_config.yaml
         - clusterresource
         - sync
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -772,7 +772,7 @@ spec:
       labels:
         app: flytepropeller
         app.kubernetes.io/name: flytepropeller
-        app.kubernetes.io/version: 0.2.36
+        app.kubernetes.io/version: 0.2.45
     spec:
       containers:
       - args:
@@ -785,7 +785,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: docker.io/lyft/flytepropeller:v0.2.36
+        image: docker.io/lyft/flytepropeller:v0.2.45
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -1031,7 +1031,7 @@ spec:
             - /etc/flyte/config/flyteadmin_config.yaml
             - clusterresource
             - sync
-            image: docker.io/lyft/flyteadmin:v0.2.7
+            image: docker.io/lyft/flyteadmin:v0.2.8
             imagePullPolicy: IfNotPresent
             name: sync-cluster-resources
             volumeMounts:

diff --git a/end2end/tests/endtoend.yaml b/end2end/tests/endtoend.yaml
@@ -11,7 +11,7 @@ spec:
     command:
     - bash
     - -c
-    image: docker.io/lyft/flytetester:v0.1.5
+    image: docker.io/lyft/flytetester:v0.1.6
     imagePullPolicy: IfNotPresent
     name: flytetester
     resources:

diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml
@@ -16,7 +16,7 @@ spec:
       labels:
         app: flyteadmin
         app.kubernetes.io/name: flyteadmin
-        app.kubernetes.io/version: 0.2.7
+        app.kubernetes.io/version: 0.2.8
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "10254"
@@ -31,15 +31,15 @@ spec:
           name: flyte-admin-config
       initContainers:
       - name: run-migrations
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"]
         volumeMounts:
         - name: config-volume
           mountPath: /etc/flyte/config
       containers:
       - name: flyteadmin
-        image: docker.io/lyft/flyteadmin:v0.2.7
+        image: docker.io/lyft/flyteadmin:v0.2.8
         imagePullPolicy: IfNotPresent
         command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "serve"]
         ports:

diff --git a/kustomize/base/console/deployment.yaml b/kustomize/base/console/deployment.yaml
@@ -16,14 +16,14 @@ spec:
       labels:
         app: flyteconsole
         app.kubernetes.io/name: flyteconsole
-        app.kubernetes.io/version: 0.5.2
+        app.kubernetes.io/version: 0.5.3
     spec:
       volumes:
       - name: shared-data
         emptyDir: {}
       containers:
       - name: flyteconsole
-        image: docker.io/lyft/flyteconsole:v0.5.2
+        image: docker.io/lyft/flyteconsole:v0.5.3
         # args: []
         ports:
         - containerPort: 8080

diff --git a/kustomize/base/propeller/deployment.yaml b/kustomize/base/propeller/deployment.yaml
@@ -15,7 +15,7 @@ spec:
       labels:
         app: flytepropeller
         app.kubernetes.io/name: flytepropeller
-        app.kubernetes.io/version: 0.2.36
+        app.kubernetes.io/version: 0.2.45
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "10254"
@@ -31,7 +31,7 @@ spec:
           name: flyte-plugin-config
       containers:
       - name: flytepropeller
-        image: docker.io/lyft/flytepropeller:v0.2.36
+        image: docker.io/lyft/flytepropeller:v0.2.45
         command:
         - flytepropeller
         args: