Merge remote-tracking branch 'upstream/master' into offload-literal-u…

…pstream-2
flyteorg · Sep 6, 2024 · 648b920 · 648b920
2 parents 114e004 + a058fd1
commit 648b920
Show file tree

Hide file tree

Showing 234 changed files with 1,692 additions and 1,909 deletions.
diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md
@@ -25,7 +25,6 @@ Chart for basic single Flyte executable deployment
 | configuration.agentService.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` |  |
 | configuration.agentService.defaultAgent.insecure | bool | `true` |  |
 | configuration.agentService.defaultAgent.timeouts.GetTask | string | `"10s"` |  |
-| configuration.agentService.supportedTaskTypes[0] | string | `"default_task"` |  |
 | configuration.annotations | object | `{}` |  |
 | configuration.auth.authorizedUris | list | `[]` |  |
 | configuration.auth.clientSecretsExternalSecretRef | string | `""` |  |

diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml
@@ -168,8 +168,6 @@ configuration:
       timeouts:
         GetTask: 10s
       defaultTimeout: 10s
-    supportedTaskTypes:
-      - default_task
   # externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte
   # If set, no Flyte configuration will be generated by this chart
   externalConfigMap: ""

diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md
@@ -195,11 +195,11 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out |
 | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment |
 | flyteagent.enabled | bool | `false` |  |
-| flyteagent.plugin_config.plugins.agent-service | object | `{"defaultAgent":{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true},"supportedTaskTypes":["sensor"]}` | Agent service configuration for propeller. |
+| flyteagent.plugin_config.plugins.agent-service | object | `{"defaultAgent":{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true},"supportedTaskTypes":[]}` | Agent service configuration for propeller. |
 | flyteagent.plugin_config.plugins.agent-service.defaultAgent | object | `{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true}` | The default agent service to use for plugin tasks. |
 | flyteagent.plugin_config.plugins.agent-service.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | The agent service endpoint propeller should connect to. |
 | flyteagent.plugin_config.plugins.agent-service.defaultAgent.insecure | bool | `true` | Whether the connection from propeller to the agent service should use TLS. |
-| flyteagent.plugin_config.plugins.agent-service.supportedTaskTypes | list | `["sensor"]` | The task types supported by the default agent. |
+| flyteagent.plugin_config.plugins.agent-service.supportedTaskTypes | list | `[]` | The task types supported by the default agent. As of #5460 these are discovered automatically and don't need to be configured. |
 | flyteagent.podLabels | object | `{}` | Labels for flyteagent pods |
 | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment |
 | flyteconsole.enabled | bool | `true` |  |

diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml
@@ -287,11 +287,9 @@ flyteagent:
           endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000"
           # -- Whether the connection from propeller to the agent service should use TLS.
           insecure: true
-        # -- The task types supported by the default agent.
-        supportedTaskTypes:
-        - sensor
-        # -- Uncomment to enable task type that uses Flyte Agent
-        # - bigquery_query_job_task
+        # -- The task types supported by the default agent. As of #5460 these are discovered automatically and don't
+        # need to be configured.
+        supportedTaskTypes: []
   # -- Labels for flyteagent pods
   podLabels: {}
 

diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml
@@ -485,8 +485,6 @@ data:
           insecure: true
           timeouts:
             GetTask: 10s
-        supportedTaskTypes:
-        - default_task
   002-database.yaml: |
     database:
       postgres:
@@ -818,7 +816,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: cWlOc1c1bnl5ZGI3YTlzSw==
+  haSharedSecret: WVl4Y1pFNm1JTkxOMjJpZQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1249,7 +1247,7 @@ spec:
     metadata:
       annotations:
         checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035
-        checksum/configuration: 6e8a4cc6177037f26cee65d09c37c010437ea3f0989a2a2dfef380fed9f468c2
+        checksum/configuration: f746817691b502fb50f04992e148847f89c0a32d7df822dbda7f1f5fdf84f420
         checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914
       labels:
         app.kubernetes.io/component: flyte-binary
@@ -1415,7 +1413,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 7f8247a0b84f43018fdf11a598132b8a67ed9fde6573ffce801b725a6f955012
+        checksum/secret: f0211dde23276e14b7254edd85e3afd32044562e409b56d79b27202e23dc224c
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml
@@ -798,7 +798,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: UUxqaW5SeGlBbFNlQzVoag==
+  haSharedSecret: Q3BDekMwb0JHM1pxaXAycg==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1362,7 +1362,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: bea0c8f293b54e309a353e0e8563e709ad817d372d2b1dce1114188693aa3f12
+        checksum/secret: 65e52b61d9d5af16c4217a1ea9f625d8d18e3251ede28f012cc14ef665412685
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml
@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: ZmdJNWs5RUg4cWNVTVBzRw==
+  haSharedSecret: RU9mMGpZWGNCcnp4cEJ3bg==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -934,7 +934,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: a896f2c43dff6c05c154b51e4c9ec21c9e2f03ecaf4c1fed045d84523219cf63
+        checksum/secret: 90ba33230f60ed1ee81e3088a4c88b0c9408c36ad5d40270cf05f503149f25a8
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docs/deployment/agents/airflow.rst b/docs/deployment/agents/airflow.rst
@@ -21,7 +21,7 @@ Specify agent configuration
         kubectl edit configmap flyte-sandbox-config -n flyte
 
       .. code-block:: yaml
-        :emphasize-lines: 7,11,16
+        :emphasize-lines: 7,11
 
         tasks:
           task-plugins:
@@ -35,11 +35,6 @@ Specify agent configuration
               - container_array: k8s-array
               - airflow: agent-service
 
-        plugins:
-          agent-service:
-            supportedTaskTypes:
-            - airflow
-
     .. group-tab:: Flyte core
 
       Create a file named ``values-override.yaml`` and add the following configuration to it.
@@ -63,10 +58,6 @@ Specify agent configuration
                   sidecar: sidecar
                   container_array: k8s-array
                   airflow: agent-service
-            plugins:
-              agent-service:
-                supportedTaskTypes:
-                - airflow
 
 
 Upgrade the Flyte Helm release

diff --git a/docs/deployment/agents/bigquery.rst b/docs/deployment/agents/bigquery.rst
@@ -29,7 +29,7 @@ Specify agent configuration
         kubectl edit configmap flyte-sandbox-config -n flyte
 
       .. code-block:: yaml
-        :emphasize-lines: 7,11,16
+        :emphasize-lines: 7,11
   
         tasks:
           task-plugins:
@@ -42,11 +42,6 @@ Specify agent configuration
               - container: container
               - container_array: k8s-array
               - bigquery_query_job_task: agent-service
-        
-        plugins:
-          agent-service:
-            supportedTaskTypes:
-            - bigquery_query_job_task
 
     .. group-tab:: Flyte core
 
@@ -71,10 +66,6 @@ Specify agent configuration
                   sidecar: sidecar
                   container_array: k8s-array
                   bigquery_query_job_task: agent-service
-            plugins:
-              agent-service:
-                supportedTaskTypes:
-                - bigquery_query_job_task
 
 Ensure that the propeller has the correct service account for BigQuery.
 

diff --git a/docs/deployment/agents/chatgpt.rst b/docs/deployment/agents/chatgpt.rst
@@ -20,7 +20,7 @@ Specify agent configuration
         kubectl edit configmap flyte-sandbox-config -n flyte
 
       .. code-block:: yaml
-        :emphasize-lines: 7,11,16
+        :emphasize-lines: 7,11
 
         tasks:
           task-plugins:
@@ -36,8 +36,6 @@ Specify agent configuration
 
         plugins:
           agent-service:
-            supportedTaskTypes:
-            - chatgpt
             # Configuring the timeout is optional.
             # Tasks like using ChatGPT with a large model might require a longer time, 
             # so we have the option to adjust the timeout setting here.
@@ -70,8 +68,6 @@ Specify agent configuration
                   chatgpt: agent-service
             plugins:
               agent-service:
-                supportedTaskTypes:
-                - chatgpt
                 # Configuring the timeout is optional.
                 # Tasks like using ChatGPT with a large model might require a longer time, 
                 # so we have the option to adjust the timeout setting here.

diff --git a/docs/deployment/agents/databricks.rst b/docs/deployment/agents/databricks.rst
@@ -139,7 +139,7 @@ Specify agent configuration
           kubectl edit configmap flyte-sandbox-config -n flyte
 
         .. code-block:: yaml
-          :emphasize-lines: 7,12,16
+          :emphasize-lines: 7,12
 
           tasks:
             task-plugins:
@@ -153,17 +153,13 @@ Specify agent configuration
                 - sidecar
                 - k8s-array
                 - agent-service
-          plugins:
-            agent-service:
-              supportedTaskTypes:
-              - spark
 
       .. group-tab:: Helm chart
 
         Edit the relevant YAML file to specify the plugin.
 
         .. code-block:: yaml
-          :emphasize-lines: 7,11,15
+          :emphasize-lines: 7,11
 
           tasks:
             task-plugins:
@@ -176,17 +172,13 @@ Specify agent configuration
                 - container: container
                 - container_array: k8s-array
                 - spark: agent-service
-            plugins:
-              agent-service:
-                supportedTaskTypes:
-                - spark
 
   .. group-tab:: Flyte core
 
     Create a file named ``values-override.yaml`` and add the following config to it:
 
     .. code-block:: yaml
-      :emphasize-lines: 9,14-17
+      :emphasize-lines: 9
 
         enabled_plugins:
           tasks:
@@ -201,10 +193,6 @@ Specify agent configuration
                 sidecar: sidecar
                 container_array: k8s-array
                 spark: agent-service
-          plugins:
-            agent-service:
-              supportedTaskTypes:
-              - spark
 
 Add the Databricks access token
 -------------------------------

diff --git a/docs/deployment/agents/mmcloud.rst b/docs/deployment/agents/mmcloud.rst
@@ -71,8 +71,6 @@ Enable the MMCloud agent by adding the following config to the relevant YAML fil
         mmcloud-agent:
           endpoint: <AGENT_ENDPOINT>
           insecure: true
-      supportedTaskTypes:
-      - mmcloud_task
       agentForTaskTypes:
       - mmcloud_task: mmcloud-agent
 

diff --git a/docs/deployment/agents/openai_batch.rst b/docs/deployment/agents/openai_batch.rst
@@ -19,7 +19,7 @@ Specify agent configuration
         kubectl edit configmap flyte-sandbox-config -n flyte
 
       .. code-block:: yaml
-        :emphasize-lines: 7,11,15
+        :emphasize-lines: 7,11
 
         tasks:
           task-plugins:
@@ -32,17 +32,13 @@ Specify agent configuration
               - container: container
               - container_array: k8s-array
               - openai-batch: agent-service
-        plugins:
-          agent-service:
-            supportedTaskTypes:
-            - openai-batch
 
     .. group-tab:: Flyte core
 
       Create a file named ``values-override.yaml`` and add the following configuration to it:
 
       .. code-block:: yaml
-        :emphasize-lines: 9,14,18
+        :emphasize-lines: 9,14
 
         configmap:
           enabled_plugins:
@@ -58,10 +54,6 @@ Specify agent configuration
                   sidecar: sidecar
                   container_array: k8s-array
                   openai-batch: agent-service
-            plugins:
-              agent-service:
-                supportedTaskTypes:
-                - openai-batch
 
 Add the OpenAI API token
 ------------------------

diff --git a/docs/deployment/agents/sagemaker_inference.rst b/docs/deployment/agents/sagemaker_inference.rst
@@ -19,7 +19,7 @@ Specify agent configuration
         kubectl edit configmap flyte-sandbox-config -n flyte
 
       .. code-block:: yaml
-        :emphasize-lines: 7,11-12,16-17
+        :emphasize-lines: 7,11-12
 
         tasks:
           task-plugins:
@@ -33,18 +33,13 @@ Specify agent configuration
               - container_array: k8s-array
               - boto: agent-service
               - sagemaker-endpoint: agent-service
-        plugins:
-          agent-service:
-            supportedTaskTypes:
-            - boto
-            - sagemaker-endpoint
 
     .. group-tab:: Flyte core
 
       Create a file named ``values-override.yaml`` and add the following configuration to it:
 
       .. code-block:: yaml
-        :emphasize-lines: 9,14-15,19-20
+        :emphasize-lines: 9,14-15
 
         configmap:
           enabled_plugins:
@@ -61,11 +56,6 @@ Specify agent configuration
                   container_array: k8s-array
                   boto: agent-service
                   sagemaker-endpoint: agent-service
-            plugins:
-              agent-service:
-                supportedTaskTypes:
-                - boto
-                - sagemaker-endpoint
 
 AWS credentials
 ---------------

diff --git a/docs/deployment/agents/sensor.rst b/docs/deployment/agents/sensor.rst
@@ -56,7 +56,7 @@ Enable the sensor agent by adding the following config to the relevant YAML file
         kubectl edit configmap flyte-sandbox-config -n flyte
 
       .. code-block:: yaml
-        :emphasize-lines: 7,11,16
+        :emphasize-lines: 7,11
   
         tasks:
           task-plugins:
@@ -69,11 +69,6 @@ Enable the sensor agent by adding the following config to the relevant YAML file
               - container: container
               - container_array: k8s-array
               - sensor: agent-service
-        
-        plugins:
-          agent-service:
-            supportedTaskTypes:
-            - sensor
 
     .. group-tab:: Flyte core
 
@@ -98,10 +93,6 @@ Enable the sensor agent by adding the following config to the relevant YAML file
                   sidecar: sidecar
                   container_array: k8s-array
                   sensor: agent-service
-            plugins:
-              agent-service:
-                supportedTaskTypes:
-                - sensor
 
 
 Upgrade the deployment