Trim sandbox

deployment/test/flyte_generated.yaml

@@ -580,14 +580,16 @@ data:
           rate: 10
           capacity: 100
   enabled_plugins.yaml: |
-    ##
-    # TODO  It is important to enable the plugins that you want to deploy here.
     tasks:
       task-plugins:
         enabled-plugins:
           - container
           - sidecar
           - k8s-array
+        default-for-task-types:
+          container: container
+          sidecar: sidecar
+          container_array: k8s-array
   k8s.yaml: |
     plugins:
       k8s:
@@ -601,6 +603,10 @@ data:
     logger:
       show-source: true
       level: 2
+  noop_resource_manager.yaml: |
+    propeller:
+      resourcemanager:
+        type: noop
   resource_manager.yaml: |
     propeller:
       resourcemanager:
@@ -629,7 +635,7 @@ data:
         kubernetes-url: "http://localhost:30082"
 kind: ConfigMap
 metadata:
-  name: flyte-propeller-config-45fc6m5292
+  name: flyte-propeller-config-dg8ghd754d
   namespace: flyte
 ---
 apiVersion: v1
@@ -724,23 +730,6 @@ spec:
   selector:
     app: postgres
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: redis-resource-manager
-  name: redis-resource-manager
-  namespace: flyte
-spec:
-  ports:
-  - name: redis
-    port: 6379
-    protocol: TCP
-    targetPort: redis
-  selector:
-    app: redis-resource-manager
-  type: ClusterIP
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -964,7 +953,7 @@ spec:
         - sh
         - -c
         - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done;
-        image: postgres:10.1
+        image: postgres:10.16-alpine
         name: check-db-ready
       - command:
         - flyteadmin
@@ -988,9 +977,8 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - seed-projects
-        - flytesnacks
         - flytetester
-        - flyteexamples
+        - flytesnacks
         image: ghcr.io/lyft/flyteadmin:v0.3.29
         imagePullPolicy: IfNotPresent
         name: seed-projects
@@ -1074,7 +1062,7 @@ spec:
       serviceAccountName: flytepropeller
       volumes:
       - configMap:
-          name: flyte-propeller-config-45fc6m5292
+          name: flyte-propeller-config-dg8ghd754d
         name: config-volume
 ---
 apiVersion: apps/v1
@@ -1127,7 +1115,10 @@ spec:
         app: postgres
     spec:
       containers:
-      - image: postgres:10.1
+      - env:
+        - name: POSTGRES_HOST_AUTH_METHOD
+          value: trust
+        image: postgres:10.16-alpine
         name: postgres
         ports:
         - containerPort: 5432
@@ -1139,66 +1130,6 @@ spec:
       - emptyDir: {}
         name: postgres-storage
 ---
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: redis
-  namespace: flyte
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: redis-resource-manager
-  serviceName: redis-resource-manager
-  template:
-    metadata:
-      labels:
-        app: redis-resource-manager
-    spec:
-      containers:
-      - env:
-        - name: REDIS_PASSWORD
-          value: mypassword
-        image: docker.io/bitnami/redis:4.0.2-r1
-        imagePullPolicy: IfNotPresent
-        livenessProbe:
-          exec:
-            command:
-            - redis-cli
-            - ping
-          failureThreshold: 3
-          initialDelaySeconds: 30
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: redis-resource-manager
-        ports:
-        - containerPort: 6379
-          name: redis
-          protocol: TCP
-        readinessProbe:
-          exec:
-            command:
-            - redis-cli
-            - ping
-          failureThreshold: 3
-          initialDelaySeconds: 5
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 1
-        resources:
-          requests:
-            cpu: 200m
-            memory: 128Mi
-        volumeMounts:
-        - mountPath: /bitnami
-          name: redis-data
-      dnsPolicy: ClusterFirst
-      restartPolicy: Always
-      volumes:
-      - emptyDir: {}
-        name: redis-data
----
 apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:

kustomize/overlays/sandbox/admin/deployment.yaml

@@ -7,47 +7,73 @@ spec:
   template:
     spec:
       volumes:
-      - name: resource-templates 
-        configMap:
-          name: clusterresource-template 
+        - name: resource-templates
+          configMap:
+            name: clusterresource-template
       initContainers:
-      - name: check-db-ready
-        image: postgres:10.1
-        command: ['sh', '-c',
-          'until pg_isready -h postgres -p 5432;
-          do echo waiting for database; sleep 2; done;']
-      - name: run-migrations
-        image: flyteadmin:v0.3.29
-        imagePullPolicy: IfNotPresent
-        command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml",
-                  "migrate", "run"]
-        volumeMounts:
-        - name: config-volume
-          mountPath: /etc/flyte/config
-      - name: seed-projects
-        image: flyteadmin:v0.3.29
-        imagePullPolicy: IfNotPresent
-        command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml",
-                  "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"]
-        volumeMounts:
-        - name: config-volume
-          mountPath: /etc/flyte/config
-      - name: sync-cluster-resources
-        image: flyteadmin:v0.3.29
-        imagePullPolicy: IfNotPresent
-        command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"]
-        volumeMounts:
-        - name: resource-templates 
-          mountPath: /etc/flyte/clusterresource/templates
-        - name: config-volume
-          mountPath: /etc/flyte/config
+        - name: check-db-ready
+          image: postgres:10.16-alpine
+          command: [
+              "sh",
+              "-c",
+              "until pg_isready -h postgres -p 5432;
+              do echo waiting for database; sleep 2; done;",
+            ]
+        - name: run-migrations
+          image: flyteadmin:v0.3.29
+          imagePullPolicy: IfNotPresent
+          command:
+            [
+              "flyteadmin",
+              "--logtostderr",
+              "--config",
+              "/etc/flyte/config/*.yaml",
+              "migrate",
+              "run",
+            ]
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/flyte/config
+        - name: seed-projects
+          image: flyteadmin:v0.3.29
+          imagePullPolicy: IfNotPresent
+          command:
+            [
+              "flyteadmin",
+              "--logtostderr",
+              "--config",
+              "/etc/flyte/config/*.yaml",
+              "migrate",
+              "seed-projects",
+              "flytesnacks",
+            ]
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/flyte/config
+        - name: sync-cluster-resources
+          image: flyteadmin:v0.3.29
+          imagePullPolicy: IfNotPresent
+          command:
+            [
+              "flyteadmin",
+              "--logtostderr",
+              "--config",
+              "/etc/flyte/config/*.yaml",
+              "clusterresource",
+              "sync",
+            ]
+          volumeMounts:
+            - name: resource-templates
+              mountPath: /etc/flyte/clusterresource/templates
+            - name: config-volume
+              mountPath: /etc/flyte/config
       containers:
-      - name: flyteadmin
-        resources:
-          limits:
-            memory: "200Mi"
-            cpu: "0.1"
-            ephemeral-storage: "100Mi"
+        - name: flyteadmin
+          resources:
+            limits:
+              memory: "200Mi"
+              cpu: "0.1"
+              ephemeral-storage: "100Mi"
 ---
 apiVersion: v1
 kind: Service
@@ -56,7 +82,7 @@ metadata:
   namespace: flyte
 spec:
   ports:
-  - name: redoc
-    protocol: TCP
-    port: 87
-    targetPort: 8087
+    - name: redoc
+      protocol: TCP
+      port: 87
+      targetPort: 8087

kustomize/overlays/sandbox/config/propeller/enabled_plugins.yaml

@@ -3,12 +3,8 @@ tasks:
     enabled-plugins:
       - container
       - sidecar
-      - spark
       - k8s-array
-      - pytorch
     default-for-task-types:
       container: container
       sidecar: sidecar
-      spark: spark
       container_array: k8s-array
-      pytorch: pytorch

kustomize/overlays/sandbox/config/propeller/noop_resource_manager.yaml

@@ -0,0 +1,3 @@
+propeller:
+  resourcemanager:
+    type: noop

kustomize/overlays/sandbox/dependencies/database/database.yaml

@@ -17,22 +17,25 @@ spec:
         - name: postgres-storage
           emptyDir: {}
       containers:
-      - image: "postgres:10.1"
-        name: postgres
-        ports:
-        - containerPort: 5432
+        - image: postgres
           name: postgres
-        volumeMounts:
-        - name: postgres-storage
-          mountPath: /var/lib/postgresql/data
+          env:
+            - name: POSTGRES_HOST_AUTH_METHOD
+              value: trust
+          ports:
+            - containerPort: 5432
+              name: postgres
+          volumeMounts:
+            - name: postgres-storage
+              mountPath: /var/lib/postgresql/data
 ---
- apiVersion: v1
- kind: Service
- metadata:
-   name: postgres
-   namespace: flyte
- spec:
-   ports:
-   - port: 5432
-   selector:
-     app: postgres
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: flyte
+spec:
+  ports:
+    - port: 5432
+  selector:
+    app: postgres

kustomize/overlays/sandbox/kustomization.yaml

@@ -17,13 +17,11 @@ bases:
 # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's
 # Add node ports for ease of use locally
 resources:
-  - ../../base/operators/spark
-  - ../../base/operators/kfoperators/pytorch
   - ./dependencies/database
   - ./dependencies/storage
-  - ./dependencies/redis
   - ./dependencies/contour_ingress_controller
   - ./dependencies/nodeport-services.yaml
+  - ../../base/namespace
 
 patchesStrategicMerge:
   - admin/deployment.yaml
@@ -38,16 +36,13 @@ configMapGenerator:
   - behavior: merge
     files:
       - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml
-      - ./config/clusterresource-templates/ad_spark-role.yaml
-      - ./config/clusterresource-templates/ae_spark-service-account.yaml
-      - ./config/clusterresource-templates/af_spark-role-binding.yaml
     name: clusterresource-template
   - behavior: merge
     files:
       - ./config/propeller/enabled_plugins.yaml
+      - ./config/propeller/noop_resource_manager.yaml
       - ./config/propeller/plugins/k8s.yaml
       - ./config/propeller/plugins/qubole.yaml
-      - ./config/propeller/plugins/spark.yaml
       - ./config/propeller/plugins/task_logs.yaml
       - ./config/common/storage.yaml
       - ./config/common/logger.yaml
@@ -77,3 +72,6 @@ images:
   - name: flytepropeller # match images with this name
     newTag: v0.5.13 # override the tag
     newName: ghcr.io/flyteorg/flytepropeller # override the name
+  # Override postgres image to use alpine based (rather smaller) docker image
+  - name: postgres
+    newTag: 10.16-alpine