Agent Metadata Servicer (#4511)

Signed-off-by: Future Outlier <[email protected]> Signed-off-by: Future-Outlier <[email protected]> Signed-off-by: Kevin Su <[email protected]> Co-authored-by: Future Outlier <[email protected]> Co-authored-by: Kevin Su <[email protected]> Co-authored-by: Kevin Su <[email protected]> Co-authored-by: Haytham Abuelfutuh <[email protected]>
flyteorg · Dec 20, 2023 · 3648440 · 3648440
1 parent 6c6656c
commit 3648440
Show file tree

Hide file tree

Showing 43 changed files with 9,567 additions and 638 deletions.
diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml
@@ -123,7 +123,7 @@ jobs:
     env:
       FLYTESNACKS_PRIORITIES: "P0"
       FLYTESNACKS_VERSION: ""
-    timeout-minutes: 20
+    timeout-minutes: 60
     needs: [build-and-push-single-binary-image]
     steps:
       - name: Set latest Flytesnacks release

diff --git a/charts/flyte-binary/templates/configmap.yaml b/charts/flyte-binary/templates/configmap.yaml
@@ -61,16 +61,9 @@ data:
       k8s-array:
         logs:
           config: {{- include "flyte-binary.configuration.logging.plugins" . | nindent 12 }}
-      {{- if .Values.configuration.agentService }}
+      {{- if .Values.flyteagent.enabled }}
       agent-service:
       {{- tpl ( .Values.configuration.agentService | toYaml ) . | nindent 8 }}
-      {{- else if .Values.flyteagent.enabled }}
-      agent-service:
-        defaultAgent:
-          endpoint: {{ include "flyteagent.name" . }}:{{ include "flyteagent.servicePort" . }}
-          insecure: true
-        supportedTaskTypes:
-        - bigquery_query_job_task
       {{- end }}
   002-database.yaml: |
     {{- with .Values.configuration.database }}

diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml
@@ -168,7 +168,6 @@ configuration:
       timeouts:
         GetTask: 10s
       defaultTimeout: 10s
-  #   Uncomment and modify to include configuration for Flyte Agent
     supportedTaskTypes:
       - default_task
   # externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte

diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml
@@ -123,15 +123,6 @@ data:
             kubernetes-enabled: false
             cloudwatch-enabled: false
             stackdriver-enabled: false
-      agent-service:
-        defaultAgent:
-          defaultTimeout: 10s
-          endpoint: dns:///flyteagent.flyte.svc.cluster.local:8000
-          insecure: true
-          timeouts:
-            GetTask: 10s
-        supportedTaskTypes:
-        - default_task
   002-database.yaml: |
     database:
       postgres:
@@ -367,7 +358,7 @@ spec:
         app.kubernetes.io/instance: flyte
         app.kubernetes.io/component: flyte-binary
       annotations:
-        checksum/configuration: 51494d8c2f3a6df2711d789217155f1e1b60e782dcbfd4564bfcccda24700576
+        checksum/configuration: 94a584c983cfe1077f02569ae00f81c4fddae4505901006eda76d9338bd3dda2
         checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929
         checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae
     spec:

diff --git a/...ndled/kustomize/complete-agent/patch.yaml → ...omplete-agent/add-agent-secret-patch.yaml b/...ndled/kustomize/complete-agent/patch.yaml → ...omplete-agent/add-agent-secret-patch.yaml
diff --git a/docker/sandbox-bundled/kustomize/complete-agent/kustomization.yaml b/docker/sandbox-bundled/kustomize/complete-agent/kustomization.yaml
@@ -4,12 +4,13 @@ helmCharts:
 - name: flyte-sandbox
   releaseName: flyte-sandbox
   namespace: flyte
-- name: flyteagent
-  releaseName: flyteagent
-  namespace: flyte
+  valuesInline:
+    flyte-binary:
+      flyteagent:
+        enabled: True
 namespace: flyte
 resources:
 - ../namespace.yaml
 
 patchesStrategicMerge:
-  - patch.yaml
+  - add-agent-secret-patch.yaml
diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml
@@ -45,7 +45,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   labels:
-    app.kubernetes.io/instance: flyteagent
+    app.kubernetes.io/instance: flyte-sandbox
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: flyteagent
     helm.sh/chart: flyteagent-v0.1.10
@@ -816,7 +816,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: ZnNFaHlpM01FZjRaQmNJSA==
+  haSharedSecret: cHNOS0lPZmw0andDczdaaA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1125,7 +1125,7 @@ metadata:
   annotations:
     projectcontour.io/upstream-protocol.h2c: grpc
   labels:
-    app.kubernetes.io/instance: flyteagent
+    app.kubernetes.io/instance: flyte-sandbox
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: flyteagent
     helm.sh/chart: flyteagent-v0.1.10
@@ -1138,7 +1138,7 @@ spec:
     protocol: TCP
     targetPort: agent-grpc
   selector:
-    app.kubernetes.io/instance: flyteagent
+    app.kubernetes.io/instance: flyte-sandbox
     app.kubernetes.io/name: flyteagent
   type: ClusterIP
 ---
@@ -1411,7 +1411,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 727daf2e5024ea4e541b87b717e6f9a8706552e0ae461f82b2e5260ad564766c
+        checksum/secret: ce47bbc1f3ca2d8673128f8efd226bd9c7ac4f32a6ded68e54e2ebb3a843065b
       labels:
         app: docker-registry
         release: flyte-sandbox
@@ -1722,7 +1722,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    app.kubernetes.io/instance: flyteagent
+    app.kubernetes.io/instance: flyte-sandbox
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: flyteagent
     helm.sh/chart: flyteagent-v0.1.10
@@ -1732,12 +1732,12 @@ spec:
   replicas: 1
   selector:
     matchLabels:
-      app.kubernetes.io/instance: flyteagent
+      app.kubernetes.io/instance: flyte-sandbox
       app.kubernetes.io/name: flyteagent
   template:
     metadata:
       labels:
-        app.kubernetes.io/instance: flyteagent
+        app.kubernetes.io/instance: flyte-sandbox
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: flyteagent
         helm.sh/chart: flyteagent-v0.1.10

diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml
@@ -465,15 +465,6 @@ data:
             kubernetes-template-uri: http://localhost:30080/kubernetes-dashboard/#/log/{{.namespace }}/{{ .podName }}/pod?namespace={{ .namespace }}
             cloudwatch-enabled: false
             stackdriver-enabled: false
-      agent-service:
-        defaultAgent:
-          defaultTimeout: 10s
-          endpoint: dns:///flyteagent.flyte.svc.cluster.local:8000
-          insecure: true
-          timeouts:
-            GetTask: 10s
-        supportedTaskTypes:
-        - default_task
   002-database.yaml: |
     database:
       postgres:
@@ -805,7 +796,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: UlI4aW9PaVFzcFpyRzhkTA==
+  haSharedSecret: RjR3Qnc0dWtGbHE3S3pnbA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1203,7 +1194,7 @@ spec:
     metadata:
       annotations:
         checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035
-        checksum/configuration: 6755bdc74789d7e2a80161a9f811e41ea3c0cf12bab5cb0820c7c8539b4b88fe
+        checksum/configuration: e2233c2adb914be363ec4fae808ed60ffb230367503f73643f011e728b853b49
         checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914
       labels:
         app.kubernetes.io/component: flyte-binary
@@ -1368,7 +1359,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: cb0b9e32c83fba67eefca5b8cd0a3375b07f5d5d29f6dab9a092f91ccf7bd200
+        checksum/secret: d027248867ca693de77339af17e89c22e7ad604b7abe616e40ca947ad0626a83
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml
@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: QzZJWkZlOTBQOUZxTUlvcA==
+  haSharedSecret: RWxBOE5BV0lIeEJ2d1c3ZA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -933,7 +933,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: dd543013ea0e17fb3c9b7be54fe7c6063e8f15ad6e0887596011ddd7f22f1540
+        checksum/secret: a2181c7387731100215f7ae94400f7330557b063f01f807d5aa2f333f266126e
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/flyteidl/clients/go/admin/mocks/AgentMetadataServiceClient.go b/flyteidl/clients/go/admin/mocks/AgentMetadataServiceClient.go