chore(k8s): update k8s smoketest files (#35)

cryostatio · Jul 25, 2023 · 685fa50 · 685fa50
1 parent dc0f932
commit 685fa50
Show file tree

Hide file tree

Showing 32 changed files with 454 additions and 126 deletions.
diff --git a/README.md b/README.md
@@ -145,11 +145,54 @@ $ LD_PRELOAD=$HOME/bin/libuserhosts.so curl http://cryostat:8181
 $ LD_PRELOAD=$HOME/bin/libuserhosts.so firefox http://cryostat:8181
 ```
 
-The next testing step is to run this same container setup in k8s.
+## Smoketesting in K8s
+
+The next testing step is to run this same container setup in k8s, which will require the installation of yq, Kompose and the multiforward plugin with Krew.
+
+Installing yq:
+```bash
+$ curl -L https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o yq
+$ chmod +x yq
+$ sudo mv -i ./yq /usr/local/bin/yq
+```
+
+Installing Kompose:
+```bash
+$ curl -L https://github.com/kubernetes/kompose/releases/download/v1.30.0/kompose-linux-amd64 -o kompose
+$ chmod +x kompose
+$ sudo mv -i ./kompose /usr/local/bin/kompose
+```
+
+Installing Krew:
+```bash
+$ cd "$(mktemp -d)"
+$ OS="$(uname | tr '[:upper:]' '[:lower:]')"
+$ ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')"
+$ KREW="krew-${OS}_${ARCH}"
+$ curl -L "https://github.com/kubernetes-sigs/krew/releases/latest/download/${KREW}.tar.gz"
+$ tar zxvf "${KREW}.tar.gz"
+$ ./"${KREW}" install krew
+```
+
+`~/.bashrc` (or equivalent shell configuration)
+```bash
+$ export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"
+```
+
+Installing multiforward:
+```bash
+$ kubectl krew install multiforward
+```
+
+The steps for testing:
 
 ```bash
 $ cd smoketest/k8s
 $ sh smoketest.sh kind # if you use `kind` and want to spin up a cluster, otherwise skip this if you have another cluster accessible via `kubectl`
-$ IMAGE_REPOSITORY=$QUAY_USERNAME sh smoketest.sh generate apply
+```
+If you get an error during the 'ensuring node image' step while creating cluster "kind", manually pull the podman image by running the command `podman pull docker.io/kindest/node@IMAGE_DIGEST` where IMAGE_DIGEST is the sha256 of the image. Then rerun `sh smoketest.sh kind`.
+
+```bash
+$ sh smoketest.sh generate apply
 $ sh smoketest.sh forward # if you need to use port-forwarding to get access to the cluster's services
 ```
diff --git a/smoketest/compose/cryostat_k8s.yml b/smoketest/compose/cryostat_k8s.yml
@@ -0,0 +1,32 @@
+version: "3"
+services:
+  cryostat:
+    depends_on:
+      db:
+        condition: service_healthy
+      s3:
+        condition: service_healthy
+    image: quay.io/cryostat/cryostat3:dev
+    hostname: cryostat3
+    user: "0"
+    expose:
+      - "9091"
+    ports:
+      - "8181:8181"
+    labels:
+      kompose.service.expose: "cryostat3"
+      io.cryostat.discovery: "true"
+      io.cryostat.jmxHost: "localhost"
+      io.cryostat.jmxPort: "0"
+      io.cryostat.jmxUrl: "service:jmx:rmi:///jndi/rmi://localhost:0/jmxrmi"
+    environment:
+      CRYOSTAT_DISCOVERY_PODMAN_ENABLED: "false"
+      CRYOSTAT_DISCOVERY_JDP_ENABLED: "true"
+      JAVA_OPTS_APPEND: "-XX:+FlightRecorder -XX:StartFlightRecording=name=onstart,settings=default,disk=true,maxage=5m -Dcom.sun.management.jmxremote.autodiscovery=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9091 -Dcom.sun.management.jmxremote.rmi.port=9091 -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false"
+    restart: always
+    healthcheck:
+      test: curl --fail http://localhost:8181/health/liveness || exit 1
+      interval: 10s
+      retries: 3
+      start_period: 10s
+      timeout: 5s
diff --git a/smoketest/compose/db.yml b/smoketest/compose/db.yml
@@ -15,16 +15,17 @@ services:
     command:
       - postgres
       - -c
-      - encrypt.key=${PG_ENCRYPT_KEY:-REPLACEME}
+      - encrypt.key=REPLACEME
     hostname: db
     expose:
       - "5432"
     environment:
       POSTGRES_USER: cryostat3
       POSTGRES_PASSWORD: cryostat3
+      PG_ENCRYPT_KEY: REPLACEME
     volumes:
       - postgresql:/var/lib/postgresql/data
-    restart: unless-stopped
+    restart: always
     healthcheck:
       test: pg_isready -U cryostat3 -d cryostat3 || exit 1
       interval: 10s
@@ -41,13 +42,13 @@ services:
       - "8989:8989"
     environment:
       PGADMIN_DEFAULT_EMAIL: [email protected]
-      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin}
+      PGADMIN_DEFAULT_PASSWORD: admin
       PGADMIN_LISTEN_PORT: 8989
     volumes:
       - pgadmin:/var/lib/pgadmin
       - ./servers.json:/pgadmin4/servers.json:z
       # - ./smoketest/compose/pgpass:/pgpass:z
-    restart: unless-stopped
+    restart: always
     healthcheck:
       test: wget --no-verbose --tries=1 --spider http://localhost:8989 || exit 1
       interval: 10s

diff --git a/smoketest/compose/db_k8s.yml b/smoketest/compose/db_k8s.yml
@@ -0,0 +1,64 @@
+version: "3"
+services:
+  cryostat:
+    environment:
+      QUARKUS_HIBERNATE_ORM_DATABASE_GENERATION: drop-and-create
+      QUARKUS_DATASOURCE_DB_KIND: postgresql
+      QUARKUS_DATASOURCE_USERNAME: cryostat3
+      QUARKUS_DATASOURCE_PASSWORD: cryostat3
+      QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://db:5432/cryostat3
+  db:
+    image: quay.io/cryostat/cryostat3-db:dev
+    build: ../../db
+    entrypoint:
+      - /usr/local/bin/docker-entrypoint.sh
+    command:
+      - postgres
+      - -c
+      - encrypt.key=REPLACEME
+    hostname: db
+    expose:
+      - "5432"
+    environment:
+      POSTGRES_USER: cryostat3
+      POSTGRES_PASSWORD: cryostat3
+      PG_ENCRYPT_KEY: REPLACEME
+    volumes:
+      - postgresql:/var/lib/postgresql/data
+    restart: always
+    healthcheck:
+      test: pg_isready -U cryostat3 -d cryostat3 || exit 1
+      interval: 10s
+      retries: 3
+      start_period: 10s
+      timeout: 5s
+  db-viewer:
+    depends_on:
+      db:
+        condition: service_healthy
+    image: docker.io/dpage/pgadmin4:6
+    hostname: db-viewer
+    ports:
+      - "8989:8989"
+    labels:
+      kompose.service.expose: "pgadmin"
+    environment:
+      PGADMIN_DEFAULT_EMAIL: [email protected]
+      PGADMIN_DEFAULT_PASSWORD: admin
+      PGADMIN_LISTEN_PORT: 8989
+    volumes:
+      - pgadmin:/var/lib/pgadmin
+      # - ./smoketest/compose/pgpass:/pgpass:z
+    restart: always
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8989 || exit 1
+      interval: 10s
+      retries: 3
+      start_period: 10s
+      timeout: 5s
+
+volumes:
+  postgresql:
+    driver: local
+  pgadmin:
+    driver: local
diff --git a/smoketest/compose/s3-localstack.yml b/smoketest/compose/s3-localstack.yml
@@ -23,7 +23,7 @@ services:
       # $ awslocal s3 mb s3://archivedrecordings
     labels:
       kompose.service.expose: "s3"
-    restart: unless-stopped
+    restart: always
     healthcheck:
       test: curl --fail http://localhost:4566 || exit 1
       interval: 10s

diff --git a/smoketest/compose/s3-minio.yml b/smoketest/compose/s3-minio.yml
@@ -7,8 +7,8 @@ services:
       QUARKUS_S3_PATH_STYLE_ACCESS: "true" # needed for Minio, but if the specific S3 impl supports DNS subdomain style access then that should be preferred
       QUARKUS_S3_AWS_REGION: us-east-1
       QUARKUS_S3_AWS_CREDENTIALS_TYPE: default
-      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minioroot}
-      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minioroot}
+      AWS_ACCESS_KEY_ID: minioroot
+      AWS_SECRET_ACCESS_KEY: minioroot
   s3:
     image: docker.io/minio/minio:latest
     hostname: minio
@@ -17,15 +17,15 @@ services:
       - "9000:9000"
     command: server /data --console-address ":9001"
     environment:
-      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioroot}
-      MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY:-minioroot}
+      MINIO_ROOT_USER: minioroot
+      MINIO_ROOT_PASSWORD: minioroot
       MINIO_DEFAULT_BUCKETS: archivedrecordings
     volumes:
       - minio_data:/data
       - minio_certs:/certs
     labels:
       kompose.service.expose: "minio"
-    restart: unless-stopped
+    restart: always
     healthcheck:
       test: curl --fail http://localhost:9001 || exit 1
       interval: 10s
@@ -38,3 +38,4 @@ volumes:
     driver: local
   minio_certs:
     driver: local
+
diff --git a/smoketest/compose/sample-apps.yml b/smoketest/compose/sample-apps.yml
@@ -24,7 +24,7 @@ services:
       io.cryostat.discovery: "true"
       io.cryostat.jmxHost: "sample-app"
       io.cryostat.jmxPort: "9093"
-    restart: unless-stopped
+    restart: always
     healthcheck:
       test: curl --fail http://localhost:8081 || exit 1
       interval: 10s
@@ -54,10 +54,11 @@ services:
       CRYOSTAT_AGENT_HARVESTER_MAX_FILES: 3
       CRYOSTAT_AGENT_HARVESTER_EXIT_MAX_AGE_MS: 60000
       CRYOSTAT_AGENT_HARVESTER_EXIT_MAX_SIZE_B: 153600 # "$(echo 1024*150 | bc)"
-    restart: unless-stopped
+    restart: always
     healthcheck:
       test: curl --fail http://localhost:10010 || exit 1
       interval: 10s
       retries: 3
       start_period: 10s
       timeout: 5s
+
diff --git a/smoketest/k8s/.multiforward.yml b/smoketest/k8s/.multiforward.yml
@@ -9,9 +9,10 @@ groups:
       - displayName: MINIO
         port: 9001
         namespace: default
-        name: minio
+        name: s3
         protocol: http
       - displayName: pgadmin
         port: 8989
+        name: db-viewer
         namespace: default
         protocol: http
diff --git a/smoketest/k8s/cryostat-deployment.yaml b/smoketest/k8s/cryostat-deployment.yaml
@@ -2,13 +2,18 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    kompose.cmd: kompose convert -o /home/andrew/workspace/cryostat3/smoketest/k8s -f /home/andrew/workspace/cryostat3/smoketest/k8s/../../container-compose.yml
-    kompose.service.expose: cryostat
-    kompose.version: 1.27.0 (b0ed6a2c9)
+    io.cryostat.discovery: "true"
+    io.cryostat.jmxHost: localhost
+    io.cryostat.jmxPort: "0"
+    io.cryostat.jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:0/jmxrmi
+    kompose.cmd: kompose convert -o ./ -f ./../compose/db_k8s.yml -f ./../compose/s3-minio.yml -f ./../compose/cryostat_k8s.yml -f ./../compose/sample-apps.yml --build local
+    kompose.service.expose: cryostat3
+    kompose.version: 1.30.0 (9d8dcb518)
   creationTimestamp: null
   labels:
     io.kompose.service: cryostat
   name: cryostat
+  namespace: default
 spec:
   replicas: 1
   selector:
@@ -18,17 +23,28 @@ spec:
   template:
     metadata:
       annotations:
-        kompose.cmd: kompose convert -o /home/andrew/workspace/cryostat3/smoketest/k8s -f /home/andrew/workspace/cryostat3/smoketest/k8s/../../container-compose.yml
-        kompose.service.expose: cryostat
-        kompose.version: 1.27.0 (b0ed6a2c9)
+        io.cryostat.discovery: "true"
+        io.cryostat.jmxHost: localhost
+        io.cryostat.jmxPort: "0"
+        io.cryostat.jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:0/jmxrmi
+        kompose.cmd: kompose convert -o ./ -f ./../compose/db_k8s.yml -f ./../compose/s3-minio.yml -f ./../compose/cryostat_k8s.yml -f ./../compose/sample-apps.yml --build local
+        kompose.service.expose: cryostat3
+        kompose.version: 1.30.0 (9d8dcb518)
       creationTimestamp: null
       labels:
+        io.kompose.network/compose-default: "true"
         io.kompose.service: cryostat
     spec:
       containers:
         - env:
-            - name: CRYOSTAT_JDP_ENABLED
+            - name: AWS_ACCESS_KEY_ID
+              value: minioroot
+            - name: AWS_SECRET_ACCESS_KEY
+              value: minioroot
+            - name: CRYOSTAT_DISCOVERY_JDP_ENABLED
               value: "true"
+            - name: CRYOSTAT_DISCOVERY_PODMAN_ENABLED
+              value: "false"
             - name: JAVA_OPTS_APPEND
               value: -XX:+FlightRecorder -XX:StartFlightRecording=name=onstart,settings=default,disk=true,maxage=5m -Dcom.sun.management.jmxremote.autodiscovery=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9091 -Dcom.sun.management.jmxremote.rmi.port=9091 -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false
             - name: QUARKUS_DATASOURCE_DB_KIND
@@ -41,18 +57,36 @@ spec:
               value: cryostat3
             - name: QUARKUS_HIBERNATE_ORM_DATABASE_GENERATION
               value: drop-and-create
-            - name: QUARKUS_MINIO_ACCESS_KEY
-              value: minioroot
-            - name: QUARKUS_MINIO_SECRET_KEY
-              value: minioroot
-            - name: QUARKUS_MINIO_URL
-              value: http://minio:9000
+            - name: QUARKUS_S3_AWS_CREDENTIALS_TYPE
+              value: default
+            - name: QUARKUS_S3_AWS_REGION
+              value: us-east-1
+            - name: QUARKUS_S3_ENDPOINT_OVERRIDE
+              value: http://s3:9000
+            - name: QUARKUS_S3_PATH_STYLE_ACCESS
+              value: "true"
+            - name: STORAGE_BUCKETS_ARCHIVES_NAME
+              value: archivedrecordings
           image: quay.io/cryostat/cryostat3:dev
+          livenessProbe:
+            exec:
+              command:
+                - curl --fail http://localhost:8181/health/liveness || exit 1
+            failureThreshold: 3
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
           name: cryostat
           ports:
             - containerPort: 8181
+              hostPort: 8181
+              protocol: TCP
             - containerPort: 9091
+              hostPort: 9091
+              protocol: TCP
           resources: {}
+          securityContext:
+            runAsUser: 0
       hostname: cryostat3
       restartPolicy: Always
 status: {}
diff --git a/smoketest/k8s/cryostat-ingress.yaml b/smoketest/k8s/cryostat-ingress.yaml
@@ -2,16 +2,21 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   annotations:
-    kompose.cmd: kompose convert -o /home/andrew/workspace/cryostat3/smoketest/k8s -f /home/andrew/workspace/cryostat3/smoketest/k8s/../../container-compose.yml
-    kompose.service.expose: cryostat
-    kompose.version: 1.27.0 (b0ed6a2c9)
+    io.cryostat.discovery: "true"
+    io.cryostat.jmxHost: localhost
+    io.cryostat.jmxPort: "0"
+    io.cryostat.jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:0/jmxrmi
+    kompose.cmd: kompose convert -o ./ -f ./../compose/db_k8s.yml -f ./../compose/s3-minio.yml -f ./../compose/cryostat_k8s.yml -f ./../compose/sample-apps.yml --build local
+    kompose.service.expose: cryostat3
+    kompose.version: 1.30.0 (9d8dcb518)
   creationTimestamp: null
   labels:
     io.kompose.service: cryostat
   name: cryostat
+  namespace: default
 spec:
   rules:
-    - host: cryostat
+    - host: cryostat3
       http:
         paths:
           - backend: