diff --git a/config/samples/image_cache/cache_and_delete_image.sh b/config/samples/image_cache/cache_and_delete_image.sh index c0bf9b55..db9fa56e 100755 --- a/config/samples/image_cache/cache_and_delete_image.sh +++ b/config/samples/image_cache/cache_and_delete_image.sh @@ -21,6 +21,8 @@ # # export PASSWORD=12345678 +# set -ex + TIME=3 CACHE_TIME=6 DOMAIN=${DOMAIN:-"glance-default-single.openstack.svc"} @@ -29,7 +31,6 @@ IMAGE_NAME="myimage" KEYSTONE=$(awk '/auth_url/ {print $2}' "/etc/openstack/clouds.yaml") ADMIN_PWD=${1:-12345678} ADMIN_USER=${ADMIN_USER:-"admin"} -DEBUG=0 # this method uses distributed image import and relies on the glance cli glance="glance --os-auth-url ${KEYSTONE} \ diff --git a/config/samples/layout/split/glance_v1beta1_glance.yaml b/config/samples/layout/split/glance_v1beta1_glance.yaml index 09de547c..4173885d 100644 --- a/config/samples/layout/split/glance_v1beta1_glance.yaml +++ b/config/samples/layout/split/glance_v1beta1_glance.yaml @@ -19,6 +19,8 @@ spec: glanceAPIs: default: replicas: 1 + imageCache: + size: 2G storage: storageClass: local-storage storageRequest: 10G diff --git a/test/kuttl/tests/glance_image_cache/01-assert.yaml b/test/kuttl/tests/glance_image_cache/01-assert.yaml index 004551ce..b43b8ceb 100644 --- a/test/kuttl/tests/glance_image_cache/01-assert.yaml +++ b/test/kuttl/tests/glance_image_cache/01-assert.yaml @@ -1,12 +1,15 @@ # # Check for: # - Glance CR -# - GlanceAPI glance-single CR -# - GlanceAPI glance-api StatefulSet -# - glance-api Pod -# - glance-cleaner CronJob -# - glance-pruner CronJob -# - glance imageCache PVC +# - GlanceAPI glance-external CR +# - GlanceAPI glance-internal CR +# - GlanceAPI glance-external-api StatefulSet +# - GlanceAPI glance-internal-api StatefulSet +# - glance-external-api Pod +# - glance-internal-api Pod +# - glance-internal service +# - glance-public service +# - glance internal and public endpoints apiVersion: glance.openstack.org/v1beta1 kind: Glance @@ -18,12 +21,7 @@ spec: databaseAccount: glance glanceAPIs: default: - replicas: 1 - type: "single" - imageCache: - cleanerScheduler: '*/30 * * * *' - prunerScheduler: 1 0 * * * - size: 2G + replicas: 2 secret: osp-secret storage: storageRequest: 10G @@ -31,10 +29,28 @@ spec: apiVersion: glance.openstack.org/v1beta1 kind: GlanceAPI metadata: - name: glance-default-single + name: glance-default-external spec: - apiType: single - replicas: 1 + apiType: external + databaseAccount: glance + passwordSelectors: + service: GlancePassword + replicas: 2 + imageCache: + cleanerScheduler: '*/30 * * * *' + prunerScheduler: 1 0 * * * + size: 2G +--- +apiVersion: glance.openstack.org/v1beta1 +kind: GlanceAPI +metadata: + name: glance-default-internal +spec: + apiType: internal + databaseAccount: glance + passwordSelectors: + service: GlancePassword + replicas: 2 imageCache: cleanerScheduler: '*/30 * * * *' prunerScheduler: 1 0 * * * @@ -48,38 +64,69 @@ metadata: status: phase: Running --- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + image-cache: "true" + labels: + component: glance-api + glanceAPI: glance-default-external + owner: glance-default-external + service: glance + name: glance-cache-glance-default-external-api-0 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2G + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + image-cache: "true" + labels: + component: glance-api + glanceAPI: glance-default-internal + owner: glance-default-internal + service: glance + name: glance-cache-glance-default-internal-api-0 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2G +--- apiVersion: batch/v1 kind: CronJob metadata: - name: glance-cache-glance-default-single-0-cleaner + name: glance-cache-glance-default-external-api-0-cleaner spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 jobTemplate: metadata: labels: component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single + glanceAPI: glance-default-external + owner: glance-default-external service: glance spec: + completions: 1 + parallelism: 1 template: spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: glanceAPI - operator: In - values: - - glance-default-single - topologyKey: kubernetes.io/hostname containers: - args: - -c - /usr/bin/glance-cache-cleaner --config-dir /etc/glance/glance.conf.d command: - /bin/bash - name: glance-cache-glance-default-single-0-cleaner + name: glance-cache-glance-default-external-api-0-cleaner volumeMounts: - mountPath: /etc/glance/glance.conf.d name: image-cache-config-data @@ -89,47 +136,42 @@ spec: volumes: - name: image-cache-config-data secret: + defaultMode: 420 items: - key: 00-config.conf path: 00-config.conf - secretName: glance-default-single-config-data + secretName: glance-default-external-config-data - name: glance-cache persistentVolumeClaim: - claimName: glance-cache-glance-default-single-0 + claimName: glance-cache-glance-default-external-api-0 schedule: '*/30 * * * *' + successfulJobsHistoryLimit: 3 + suspend: false --- apiVersion: batch/v1 kind: CronJob metadata: - name: glance-cache-glance-default-single-0-pruner + name: glance-cache-glance-default-external-api-0-pruner spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 jobTemplate: metadata: labels: component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single + glanceAPI: glance-default-external + owner: glance-default-external service: glance spec: template: spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: glanceAPI - operator: In - values: - - glance-default-single - topologyKey: kubernetes.io/hostname containers: - args: - -c - /usr/bin/glance-cache-pruner --config-dir /etc/glance/glance.conf.d command: - /bin/bash - name: glance-cache-glance-default-single-0-pruner + name: glance-cache-glance-default-external-api-0-pruner volumeMounts: - mountPath: /etc/glance/glance.conf.d name: image-cache-config-data @@ -139,31 +181,14 @@ spec: volumes: - name: image-cache-config-data secret: + defaultMode: 420 items: - key: 00-config.conf path: 00-config.conf - secretName: glance-default-single-config-data + secretName: glance-default-external-config-data - name: glance-cache persistentVolumeClaim: - claimName: glance-cache-glance-default-single-0 - schedule: '1 0 * * *' ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - annotations: - image-cache: "true" - pv.kubernetes.io/bind-completed: "yes" - pv.kubernetes.io/bound-by-controller: "yes" - labels: - component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single - service: glance - name: glance-cache-glance-default-single-0 -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2G + claimName: glance-cache-glance-default-external-api-0 + schedule: 1 0 * * * + successfulJobsHistoryLimit: 3 + suspend: false diff --git a/test/kuttl/tests/glance_image_cache/01-deploy_glance.yaml b/test/kuttl/tests/glance_image_cache/01-deploy_glance.yaml index cff247a9..adb9705e 100644 --- a/test/kuttl/tests/glance_image_cache/01-deploy_glance.yaml +++ b/test/kuttl/tests/glance_image_cache/01-deploy_glance.yaml @@ -2,4 +2,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - oc kustomize ../../../../config/samples/layout/single | oc apply -n $NAMESPACE -f - + oc kustomize ../../../../config/samples/layout/split | oc apply -n $NAMESPACE -f - + - script: | + oc patch Glance -n $NAMESPACE glance --type='json' -p='[{"op": "replace", "path": "/spec/glanceAPIs/default/replicas", "value":2}]' + - script: | + oc -n $NAMESPACE kustomize --load-restrictor LoadRestrictionsNone ../../../../config/samples/openstackclient | oc -n $NAMESPACE apply -f - diff --git a/test/kuttl/tests/glance_image_cache/02-assert.yaml b/test/kuttl/tests/glance_image_cache/02-assert.yaml deleted file mode 100644 index 38d4aeb5..00000000 --- a/test/kuttl/tests/glance_image_cache/02-assert.yaml +++ /dev/null @@ -1,278 +0,0 @@ -# -# For both replicas, check for: -# - Glance CR -# - GlanceAPI glance-single CR -# - GlanceAPI glance-api StatefulSet -# - glance-api Pod -# - glance-cleaner CronJob -# - glance-pruner CronJob -# - glance imageCache PVC - -apiVersion: glance.openstack.org/v1beta1 -kind: Glance -metadata: - name: glance -spec: - serviceUser: glance - databaseInstance: openstack - databaseAccount: glance - glanceAPIs: - default: - replicas: 3 - type: "single" - imageCache: - cleanerScheduler: '*/30 * * * *' - prunerScheduler: 1 0 * * * - size: 2G ---- -apiVersion: glance.openstack.org/v1beta1 -kind: GlanceAPI -metadata: - name: glance-default-single -spec: - apiType: single - replicas: 3 - imageCache: - cleanerScheduler: '*/30 * * * *' - prunerScheduler: 1 0 * * * - size: 2G ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: glance-cache-glance-default-single-0-cleaner -spec: - jobTemplate: - metadata: - labels: - component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single - service: glance - spec: - template: - spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: glanceAPI - operator: In - values: - - glance-default-single - topologyKey: kubernetes.io/hostname - containers: - - args: - - -c - - /usr/bin/glance-cache-cleaner --config-dir /etc/glance/glance.conf.d - command: - - /bin/bash - name: glance-cache-glance-default-single-0-cleaner - volumeMounts: - - mountPath: /etc/glance/glance.conf.d - name: image-cache-config-data - readOnly: true - - mountPath: /var/lib/glance/image-cache - name: glance-cache - volumes: - - name: image-cache-config-data - secret: - items: - - key: 00-config.conf - path: 00-config.conf - secretName: glance-default-single-config-data - - name: glance-cache - persistentVolumeClaim: - claimName: glance-cache-glance-default-single-0 - schedule: '*/30 * * * *' ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: glance-cache-glance-default-single-1-cleaner -spec: - jobTemplate: - metadata: - labels: - component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single - service: glance - spec: - template: - spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: glanceAPI - operator: In - values: - - glance-default-single - topologyKey: kubernetes.io/hostname - containers: - - args: - - -c - - /usr/bin/glance-cache-cleaner --config-dir /etc/glance/glance.conf.d - command: - - /bin/bash - name: glance-cache-glance-default-single-1-cleaner - volumeMounts: - - mountPath: /etc/glance/glance.conf.d - name: image-cache-config-data - readOnly: true - - mountPath: /var/lib/glance/image-cache - name: glance-cache - volumes: - - name: image-cache-config-data - secret: - items: - - key: 00-config.conf - path: 00-config.conf - secretName: glance-default-single-config-data - - name: glance-cache - persistentVolumeClaim: - claimName: glance-cache-glance-default-single-1 - schedule: '*/30 * * * *' ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: glance-cache-glance-default-single-0-pruner -spec: - jobTemplate: - metadata: - labels: - component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single - service: glance - spec: - template: - spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: glanceAPI - operator: In - values: - - glance-default-single - topologyKey: kubernetes.io/hostname - containers: - - args: - - -c - - /usr/bin/glance-cache-pruner --config-dir /etc/glance/glance.conf.d - command: - - /bin/bash - name: glance-cache-glance-default-single-0-pruner - volumeMounts: - - mountPath: /etc/glance/glance.conf.d - name: image-cache-config-data - readOnly: true - - mountPath: /var/lib/glance/image-cache - name: glance-cache - volumes: - - name: image-cache-config-data - secret: - items: - - key: 00-config.conf - path: 00-config.conf - secretName: glance-default-single-config-data - - name: glance-cache - persistentVolumeClaim: - claimName: glance-cache-glance-default-single-0 - schedule: '1 0 * * *' ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: glance-cache-glance-default-single-1-pruner -spec: - jobTemplate: - metadata: - labels: - component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single - service: glance - spec: - template: - spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: glanceAPI - operator: In - values: - - glance-default-single - topologyKey: kubernetes.io/hostname - containers: - - args: - - -c - - /usr/bin/glance-cache-pruner --config-dir /etc/glance/glance.conf.d - command: - - /bin/bash - name: glance-cache-glance-default-single-1-pruner - volumeMounts: - - mountPath: /etc/glance/glance.conf.d - name: image-cache-config-data - readOnly: true - - mountPath: /var/lib/glance/image-cache - name: glance-cache - volumes: - - name: image-cache-config-data - secret: - items: - - key: 00-config.conf - path: 00-config.conf - secretName: glance-default-single-config-data - - name: glance-cache - persistentVolumeClaim: - claimName: glance-cache-glance-default-single-1 - schedule: '1 0 * * *' ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - annotations: - image-cache: "true" - pv.kubernetes.io/bind-completed: "yes" - pv.kubernetes.io/bound-by-controller: "yes" - labels: - component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single - service: glance - name: glance-cache-glance-default-single-0 -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2G ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - annotations: - image-cache: "true" - pv.kubernetes.io/bind-completed: "yes" - pv.kubernetes.io/bound-by-controller: "yes" - labels: - component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single - service: glance - name: glance-cache-glance-default-single-1 -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2G diff --git a/test/kuttl/tests/glance_image_cache/02-cache-image.yaml b/test/kuttl/tests/glance_image_cache/02-cache-image.yaml new file mode 100644 index 00000000..032f857a --- /dev/null +++ b/test/kuttl/tests/glance_image_cache/02-cache-image.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc -n $NAMESPACE exec -it openstackclient --stdin=false -- /bin/sh -c "DOMAIN=glance-default-external-api.$NAMESPACE.svc REPLICA=glance-default-external-api- ./scripts/cache_and_delete_image.sh" diff --git a/test/kuttl/tests/glance_image_cache/02-scale-glanceapis.yaml b/test/kuttl/tests/glance_image_cache/02-scale-glanceapis.yaml deleted file mode 100644 index edb390ce..00000000 --- a/test/kuttl/tests/glance_image_cache/02-scale-glanceapis.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - oc patch Glance -n $NAMESPACE glance --type='json' -p='[{"op": "replace", "path": "/spec/glanceAPIs/default/replicas", "value":3}]' diff --git a/test/kuttl/tests/glance_image_cache/03-assert.yaml b/test/kuttl/tests/glance_image_cache/03-assert.yaml index f7523a9a..47bdc662 100644 --- a/test/kuttl/tests/glance_image_cache/03-assert.yaml +++ b/test/kuttl/tests/glance_image_cache/03-assert.yaml @@ -1,13 +1,15 @@ # -# # Check for: # - Glance CR -# - GlanceAPI glance-single CR -# - GlanceAPI glance-api StatefulSet -# - glance-api Pod -# - glance-cleaner CronJob -# - glance-pruner CronJob -# - glance imageCache PVC +# - GlanceAPI glance-external CR +# - GlanceAPI glance-internal CR +# - GlanceAPI glance-external-api StatefulSet +# - GlanceAPI glance-internal-api StatefulSet +# - glance-external-api Pod +# - glance-internal-api Pod +# - glance-internal service +# - glance-public service +# - glance internal and public endpoints apiVersion: glance.openstack.org/v1beta1 kind: Glance @@ -20,18 +22,34 @@ spec: glanceAPIs: default: replicas: 1 - type: "single" - imageCache: - cleanerScheduler: '*/30 * * * *' - prunerScheduler: 1 0 * * * - size: 2G + secret: osp-secret + storage: + storageRequest: 10G --- apiVersion: glance.openstack.org/v1beta1 kind: GlanceAPI metadata: - name: glance-default-single + name: glance-default-external spec: - apiType: single + apiType: external + databaseAccount: glance + passwordSelectors: + service: GlancePassword + replicas: 1 + imageCache: + cleanerScheduler: '*/30 * * * *' + prunerScheduler: 1 0 * * * + size: 2G +--- +apiVersion: glance.openstack.org/v1beta1 +kind: GlanceAPI +metadata: + name: glance-default-internal +spec: + apiType: internal + databaseAccount: glance + passwordSelectors: + service: GlancePassword replicas: 1 imageCache: cleanerScheduler: '*/30 * * * *' @@ -46,38 +64,69 @@ metadata: status: phase: Running --- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + image-cache: "true" + labels: + component: glance-api + glanceAPI: glance-default-external + owner: glance-default-external + service: glance + name: glance-cache-glance-default-external-api-0 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2G + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + image-cache: "true" + labels: + component: glance-api + glanceAPI: glance-default-internal + owner: glance-default-internal + service: glance + name: glance-cache-glance-default-internal-api-0 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2G +--- apiVersion: batch/v1 kind: CronJob metadata: - name: glance-cache-glance-default-single-0-cleaner + name: glance-cache-glance-default-external-api-0-cleaner spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 jobTemplate: metadata: labels: component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single + glanceAPI: glance-default-external + owner: glance-default-external service: glance spec: + completions: 1 + parallelism: 1 template: spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: glanceAPI - operator: In - values: - - glance-default-single - topologyKey: kubernetes.io/hostname containers: - args: - -c - /usr/bin/glance-cache-cleaner --config-dir /etc/glance/glance.conf.d command: - /bin/bash - name: glance-cache-glance-default-single-0-cleaner + name: glance-cache-glance-default-external-api-0-cleaner volumeMounts: - mountPath: /etc/glance/glance.conf.d name: image-cache-config-data @@ -87,47 +136,42 @@ spec: volumes: - name: image-cache-config-data secret: + defaultMode: 420 items: - key: 00-config.conf path: 00-config.conf - secretName: glance-default-single-config-data + secretName: glance-default-external-config-data - name: glance-cache persistentVolumeClaim: - claimName: glance-cache-glance-default-single-0 + claimName: glance-cache-glance-default-external-api-0 schedule: '*/30 * * * *' + successfulJobsHistoryLimit: 3 + suspend: false --- apiVersion: batch/v1 kind: CronJob metadata: - name: glance-cache-glance-default-single-0-pruner + name: glance-cache-glance-default-external-api-0-pruner spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 jobTemplate: metadata: labels: component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single + glanceAPI: glance-default-external + owner: glance-default-external service: glance spec: template: spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: glanceAPI - operator: In - values: - - glance-default-single - topologyKey: kubernetes.io/hostname containers: - args: - -c - /usr/bin/glance-cache-pruner --config-dir /etc/glance/glance.conf.d command: - /bin/bash - name: glance-cache-glance-default-single-0-pruner + name: glance-cache-glance-default-external-api-0-pruner volumeMounts: - mountPath: /etc/glance/glance.conf.d name: image-cache-config-data @@ -137,31 +181,14 @@ spec: volumes: - name: image-cache-config-data secret: + defaultMode: 420 items: - key: 00-config.conf path: 00-config.conf - secretName: glance-default-single-config-data + secretName: glance-default-external-config-data - name: glance-cache persistentVolumeClaim: - claimName: glance-cache-glance-default-single-0 - schedule: '1 0 * * *' ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - annotations: - image-cache: "true" - pv.kubernetes.io/bind-completed: "yes" - pv.kubernetes.io/bound-by-controller: "yes" - labels: - component: glance-api - glanceAPI: glance-default-single - owner: glance-default-single - service: glance - name: glance-cache-glance-default-single-0 -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2G + claimName: glance-cache-glance-default-external-api-0 + schedule: 1 0 * * * + successfulJobsHistoryLimit: 3 + suspend: false