Trim sandbox (flyteorg#727)

* Trim Sandbox * Add namespace * Use smaller postgres * allow anonymous connections to postgres * trim * PR Comments, trim test kustomize too * More cleanup * add back flytetester project for testing * Revert the rename * revert ingress change * PR Comments
sbrunk · Feb 15, 2021 · cd4a3d5 · cd4a3d5
1 parent 9117e4c
commit cd4a3d5
Show file tree

Hide file tree

Showing 19 changed files with 1,933 additions and 10,146 deletions.
diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml
diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml
@@ -580,14 +580,16 @@ data:
           rate: 10
           capacity: 100
   enabled_plugins.yaml: |
-    ##
-    # TODO  It is important to enable the plugins that you want to deploy here.
     tasks:
       task-plugins:
         enabled-plugins:
           - container
           - sidecar
           - k8s-array
+        default-for-task-types:
+          container: container
+          sidecar: sidecar
+          container_array: k8s-array
   k8s.yaml: |
     plugins:
       k8s:
@@ -604,11 +606,7 @@ data:
   resource_manager.yaml: |
     propeller:
       resourcemanager:
-        type: redis
-        resourceMaxQuota: 10000
-        redis:
-          hostPath: redis-resource-manager:6379
-          hostKey: mypassword
+        type: noop
   storage.yaml: |+
     storage:
       type: minio
@@ -629,7 +627,7 @@ data:
         kubernetes-url: "http://localhost:30082"
 kind: ConfigMap
 metadata:
-  name: flyte-propeller-config-45fc6m5292
+  name: flyte-propeller-config-569h4b5459
   namespace: flyte
 ---
 apiVersion: v1
@@ -724,23 +722,6 @@ spec:
   selector:
     app: postgres
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: redis-resource-manager
-  name: redis-resource-manager
-  namespace: flyte
-spec:
-  ports:
-  - name: redis
-    port: 6379
-    protocol: TCP
-    targetPort: redis
-  selector:
-    app: redis-resource-manager
-  type: ClusterIP
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -964,7 +945,7 @@ spec:
         - sh
         - -c
         - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done;
-        image: postgres:10.1
+        image: postgres:10.16-alpine
         name: check-db-ready
       - command:
         - flyteadmin
@@ -988,9 +969,8 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - seed-projects
-        - flytesnacks
         - flytetester
-        - flyteexamples
+        - flytesnacks
         image: ghcr.io/lyft/flyteadmin:v0.3.29
         imagePullPolicy: IfNotPresent
         name: seed-projects
@@ -1074,7 +1054,7 @@ spec:
       serviceAccountName: flytepropeller
       volumes:
       - configMap:
-          name: flyte-propeller-config-45fc6m5292
+          name: flyte-propeller-config-569h4b5459
         name: config-volume
 ---
 apiVersion: apps/v1
@@ -1127,7 +1107,10 @@ spec:
         app: postgres
     spec:
       containers:
-      - image: postgres:10.1
+      - env:
+        - name: POSTGRES_HOST_AUTH_METHOD
+          value: trust
+        image: postgres:10.16-alpine
         name: postgres
         ports:
         - containerPort: 5432
@@ -1139,66 +1122,6 @@ spec:
       - emptyDir: {}
         name: postgres-storage
 ---
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: redis
-  namespace: flyte
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: redis-resource-manager
-  serviceName: redis-resource-manager
-  template:
-    metadata:
-      labels:
-        app: redis-resource-manager
-    spec:
-      containers:
-      - env:
-        - name: REDIS_PASSWORD
-          value: mypassword
-        image: docker.io/bitnami/redis:4.0.2-r1
-        imagePullPolicy: IfNotPresent
-        livenessProbe:
-          exec:
-            command:
-            - redis-cli
-            - ping
-          failureThreshold: 3
-          initialDelaySeconds: 30
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: redis-resource-manager
-        ports:
-        - containerPort: 6379
-          name: redis
-          protocol: TCP
-        readinessProbe:
-          exec:
-            command:
-            - redis-cli
-            - ping
-          failureThreshold: 3
-          initialDelaySeconds: 5
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 1
-        resources:
-          requests:
-            cpu: 200m
-            memory: 128Mi
-        volumeMounts:
-        - mountPath: /bitnami
-          name: redis-data
-      dnsPolicy: ClusterFirst
-      restartPolicy: Always
-      volumes:
-      - emptyDir: {}
-        name: redis-data
----
 apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:

diff --git a/kustomize/overlays/sandbox/admin/deployment.yaml b/kustomize/overlays/sandbox/admin/deployment.yaml
@@ -7,47 +7,73 @@ spec:
   template:
     spec:
       volumes:
-      - name: resource-templates 
-        configMap:
-          name: clusterresource-template 
+        - name: resource-templates
+          configMap:
+            name: clusterresource-template
       initContainers:
-      - name: check-db-ready
-        image: postgres:10.1
-        command: ['sh', '-c',
-          'until pg_isready -h postgres -p 5432;
-          do echo waiting for database; sleep 2; done;']
-      - name: run-migrations
-        image: flyteadmin:v0.3.29
-        imagePullPolicy: IfNotPresent
-        command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml",
-                  "migrate", "run"]
-        volumeMounts:
-        - name: config-volume
-          mountPath: /etc/flyte/config
-      - name: seed-projects
-        image: flyteadmin:v0.3.29
-        imagePullPolicy: IfNotPresent
-        command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml",
-                  "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"]
-        volumeMounts:
-        - name: config-volume
-          mountPath: /etc/flyte/config
-      - name: sync-cluster-resources
-        image: flyteadmin:v0.3.29
-        imagePullPolicy: IfNotPresent
-        command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"]
-        volumeMounts:
-        - name: resource-templates 
-          mountPath: /etc/flyte/clusterresource/templates
-        - name: config-volume
-          mountPath: /etc/flyte/config
+        - name: check-db-ready
+          image: postgres:10.16-alpine
+          command: [
+              "sh",
+              "-c",
+              "until pg_isready -h postgres -p 5432;
+              do echo waiting for database; sleep 2; done;",
+            ]
+        - name: run-migrations
+          image: flyteadmin:v0.3.29
+          imagePullPolicy: IfNotPresent
+          command:
+            [
+              "flyteadmin",
+              "--logtostderr",
+              "--config",
+              "/etc/flyte/config/*.yaml",
+              "migrate",
+              "run",
+            ]
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/flyte/config
+        - name: seed-projects
+          image: flyteadmin:v0.3.29
+          imagePullPolicy: IfNotPresent
+          command:
+            [
+              "flyteadmin",
+              "--logtostderr",
+              "--config",
+              "/etc/flyte/config/*.yaml",
+              "migrate",
+              "seed-projects",
+              "flytesnacks",
+            ]
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/flyte/config
+        - name: sync-cluster-resources
+          image: flyteadmin:v0.3.29
+          imagePullPolicy: IfNotPresent
+          command:
+            [
+              "flyteadmin",
+              "--logtostderr",
+              "--config",
+              "/etc/flyte/config/*.yaml",
+              "clusterresource",
+              "sync",
+            ]
+          volumeMounts:
+            - name: resource-templates
+              mountPath: /etc/flyte/clusterresource/templates
+            - name: config-volume
+              mountPath: /etc/flyte/config
       containers:
-      - name: flyteadmin
-        resources:
-          limits:
-            memory: "200Mi"
-            cpu: "0.1"
-            ephemeral-storage: "100Mi"
+        - name: flyteadmin
+          resources:
+            limits:
+              memory: "200Mi"
+              cpu: "0.1"
+              ephemeral-storage: "100Mi"
 ---
 apiVersion: v1
 kind: Service
@@ -56,7 +82,7 @@ metadata:
   namespace: flyte
 spec:
   ports:
-  - name: redoc
-    protocol: TCP
-    port: 87
-    targetPort: 8087
+    - name: redoc
+      protocol: TCP
+      port: 87
+      targetPort: 8087
diff --git a/kustomize/overlays/sandbox/config/propeller/enabled_plugins.yaml b/kustomize/overlays/sandbox/config/propeller/enabled_plugins.yaml
@@ -3,12 +3,8 @@ tasks:
     enabled-plugins:
       - container
       - sidecar
-      - spark
       - k8s-array
-      - pytorch
     default-for-task-types:
       container: container
       sidecar: sidecar
-      spark: spark
       container_array: k8s-array
-      pytorch: pytorch
diff --git a/kustomize/overlays/sandbox/config/propeller/plugins/qubole.yaml b/kustomize/overlays/sandbox/config/propeller/plugins/qubole.yaml
diff --git a/kustomize/overlays/sandbox/config/propeller/plugins/spark.yaml b/kustomize/overlays/sandbox/config/propeller/plugins/spark.yaml
diff --git a/kustomize/overlays/sandbox/config/propeller/resource_manager.yaml b/kustomize/overlays/sandbox/config/propeller/resource_manager.yaml
@@ -0,0 +1,3 @@
+propeller:
+  resourcemanager:
+    type: noop
diff --git a/kustomize/overlays/sandbox/dependencies/database/database.yaml b/kustomize/overlays/sandbox/dependencies/database/database.yaml
@@ -17,22 +17,25 @@ spec:
         - name: postgres-storage
           emptyDir: {}
       containers:
-      - image: "postgres:10.1"
-        name: postgres
-        ports:
-        - containerPort: 5432
+        - image: postgres
           name: postgres
-        volumeMounts:
-        - name: postgres-storage
-          mountPath: /var/lib/postgresql/data
+          env:
+            - name: POSTGRES_HOST_AUTH_METHOD
+              value: trust
+          ports:
+            - containerPort: 5432
+              name: postgres
+          volumeMounts:
+            - name: postgres-storage
+              mountPath: /var/lib/postgresql/data
 ---
- apiVersion: v1
- kind: Service
- metadata:
-   name: postgres
-   namespace: flyte
- spec:
-   ports:
-   - port: 5432
-   selector:
-     app: postgres
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: flyte
+spec:
+  ports:
+    - port: 5432
+  selector:
+    app: postgres