From b60467e75fd1c995b4caa024f2fb116f3e584707 Mon Sep 17 00:00:00 2001 From: Shreyas Sreenivas Date: Mon, 19 Aug 2024 10:16:57 +0530 Subject: [PATCH 1/2] PWX-38500 DRW changes Signed-off-by: Piyush Nimbalkar PWX-38500 Resources and placement support CSI and Portworx API Signed-off-by: Piyush Nimbalkar PWX-38500 Custom annotation support for CSI, Portworx API, KVDB Signed-off-by: Piyush Nimbalkar PWX-38415 Fix resources comparison when resources is reset in STC Signed-off-by: Piyush Nimbalkar PWX-38379 change pool and volume resize alerts from warning to info Docs PR: https://github.com/pure-px/portworx-documentation/pull/977 PWX-38500 Add UTs for resources, annotation and placement changes Signed-off-by: Piyush Nimbalkar Create a new 24.1.2-drw release --- Makefile | 2 +- deploy/crds/core_v1_storagecluster_crd.yaml | 332 ++ .../core_v1_storagecluster_crd.yaml | 3939 +++++++++++++++++ .../24.1.2-drw/core_v1_storagenode_crd.yaml | 191 + ...tworx-certified.clusterserviceversion.yaml | 551 +++ .../portworx/portworx.package.yaml | 4 +- deploy/operator.yaml | 2 +- drivers/storage/portworx/component/csi.go | 272 +- .../portworx/component/portworx_api.go | 118 +- drivers/storage/portworx/components_test.go | 650 ++- drivers/storage/portworx/deployment.go | 28 +- drivers/storage/portworx/deployment_test.go | 19 + .../portworx/testspec/kvdbPodCustomPort.yaml | 13 + .../portworx/testspec/kvdbPodDefault.yaml | 13 + .../testspec/portworxAPIDaemonSet.yaml | 10 + .../testspec/portworxAPIDaemonset_2_13.yaml | 20 +- pkg/apis/core/v1/storagecluster.go | 52 +- pkg/apis/core/v1/zz_generated.deepcopy.go | 166 + pkg/constants/metadata.go | 2 + pkg/controller/storagenode/storagenode.go | 44 +- .../storagenode/storagenode_test.go | 216 +- pkg/util/k8s/k8s.go | 19 +- pkg/util/util.go | 70 +- 23 files changed, 6499 insertions(+), 234 deletions(-) create mode 100644 deploy/olm-catalog/portworx/24.1.2-drw/core_v1_storagecluster_crd.yaml create mode 100644 deploy/olm-catalog/portworx/24.1.2-drw/core_v1_storagenode_crd.yaml create mode 100644 deploy/olm-catalog/portworx/24.1.2-drw/portworx-certified.clusterserviceversion.yaml diff --git a/Makefile b/Makefile index b5a9b9a5c5..677f3a85cf 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ ifndef DOCKER_HUB_REGISTRY_IMG $(warning DOCKER_HUB_REGISTRY_IMG not defined, using '$(DOCKER_HUB_REGISTRY_IMG)' instead) endif ifndef BASE_REGISTRY_IMG - BASE_REGISTRY_IMG := docker.io/portworx/px-operator-registry:24.1.0 + BASE_REGISTRY_IMG := docker.io/portworx/px-operator-registry:24.1.1 $(warning BASE_REGISTRY_IMG not defined, using '$(BASE_REGISTRY_IMG)' instead) endif diff --git a/deploy/crds/core_v1_storagecluster_crd.yaml b/deploy/crds/core_v1_storagecluster_crd.yaml index 491d1d761c..824019d1df 100644 --- a/deploy/crds/core_v1_storagecluster_crd.yaml +++ b/deploy/crds/core_v1_storagecluster_crd.yaml @@ -80,6 +80,16 @@ spec: cpu: type: string description: Requested cpu for the storage pod. + limits: + type: object + description: Limits describes the maximum amount of compute resources allowed for the storage pod. + properties: + memory: + type: string + description: Maximum memory for the storage pod. + cpu: + type: string + description: Maximum cpu for the storage pod. image: type: string description: Docker image of the storage driver. @@ -342,6 +352,30 @@ spec: description: Authentication secret is the name of Kubernetes secret containing information to authenticate with the external KVDB. It could have the username/password for basic auth, certificate information or an ACL token. + resources: + type: object + description: Specifies the compute resource requirements for the kvdb pod. + properties: + requests: + type: object + description: Requested resources for the kvdb pod. + properties: + memory: + type: string + description: Requested memory for the kvdb pod. + cpu: + type: string + description: Requested cpu for the kvdb pod. + limits: + type: object + description: Limits describes the maximum amount of compute resources allowed for the kvdb pod. + properties: + memory: + type: string + description: Maximum memory for the kvdb pod. + cpu: + type: string + description: Maximum cpu for the kvdb pod. storage: type: object description: Details of the storage used by the storage driver. @@ -3458,6 +3492,304 @@ spec: enabled: type: boolean description: Flag indicating whether CSI topology feature gate is enabled. + placement: + type: object + description: Describes placement configuration for the CSI sidecar pods. + properties: + nodeAffinity: + type: object + description: Describes node affinity scheduling rules for the CSI sidecar pods. + This is exactly the same object as Kubernetes node affinity for pods. + properties: + requiredDuringSchedulingIgnoredDuringExecution: + type: object + properties: + nodeSelectorTerms: + type: array + items: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + required: + - nodeSelectorTerms + preferredDuringSchedulingIgnoredDuringExecution: + type: array + items: + type: object + properties: + weight: + type: integer + preference: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + required: + - preference + - weight + tolerations: + type: array + description: Tolerations for the CSI sidecar pods. + The pod with this toleration attached will tolerate any taint that matches the + triple using the matching operator . + items: + type: object + properties: + effect: + type: string + description: Effect indicates the taint effect to match. Empty means match + all taint effects. When specified, allowed values are NoSchedule, + PreferNoSchedule and NoExecute. + key: + type: string + description: Key is the taint key that the toleration applies to. Empty means + match all taint keys. If the key is empty, operator must be Exists; this + combination means to match all values and all keys. + operator: + type: string + description: "Operator represents a key's relationship to the value. Valid + operators are Exists and Equal. Defaults to Equal. Exists is equivalent to + wildcard for value, so that a pod can tolerate all taints of a particular category." + value: + type: string + description: Value is the taint value the toleration matches to. If the operator + is Exists, the value should be empty, otherwise just a regular string. + tolerationSeconds: + type: integer + description: TolerationSeconds represents the period of time the toleration + (which must be of effect NoExecute, otherwise this field is ignored) tolerates + the taint. By default, it is not set, which means tolerate the taint forever + (do not evict). Zero and negative values will be treated as 0 (evict + immediately) by the system. + nodeDriverRegistrar: + type: object + description: Defines configuration for the CSI node driver registrar. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi node driver registar container. + properties: + requests: + type: object + description: Requested resources for the csi node driver registrar container. + properties: + memory: + type: string + description: Requested memory for the csi node driver registrar container. + cpu: + type: string + description: Requested cpu for the csi node driver registrar container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi node driver registrar container. + properties: + memory: + type: string + description: Maximum memory for the csi node driver registrar container. + cpu: + type: string + description: Maximum cpu for the csi node driver registrar container. + externalProvisioner: + type: object + description: Defines configuration for the CSI provisioner. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi provisioner container. + properties: + requests: + type: object + description: Requested resources for the csi provisioner container. + properties: + memory: + type: string + description: Requested memory for the csi provisioner container. + cpu: + type: string + description: Requested cpu for the csi provisioner container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi provisioner container. + properties: + memory: + type: string + description: Maximum memory for the csi provisioner container. + cpu: + type: string + description: Maximum cpu for the csi provisioner container. + snapshotter: + type: object + description: Defines configuration for the CSI snapshotter. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi snapshotter container. + properties: + requests: + type: object + description: Requested resources for the csi snapshotter container. + properties: + memory: + type: string + description: Requested memory for the csi snapshotter container. + cpu: + type: string + description: Requested cpu for the csi snapshotter container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi snapshotter container. + properties: + memory: + type: string + description: Maximum memory for the csi snapshotter container. + cpu: + type: string + description: Maximum cpu for the csi snapshotter container. + resizer: + type: object + description: Defines configuration for the CSI resizer. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi resizer container. + properties: + requests: + type: object + description: Requested resources for the csi resizer container. + properties: + memory: + type: string + description: Requested memory for the csi resizer container. + cpu: + type: string + description: Requested cpu for the csi resizer container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi resizer container. + properties: + memory: + type: string + description: Maximum memory for the csi resizer container. + cpu: + type: string + description: Maximum cpu for the csi resizer container. + snapshotController: + type: object + description: Defines configuration for the CSI snapshot controller. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi snapshot controller container. + properties: + requests: + type: object + description: Requested resources for the csi snapshot controller container. + properties: + memory: + type: string + description: Requested memory for the csi snapshot controller container. + cpu: + type: string + description: Requested cpu for the csi snapshot controller container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi snapshot controller container. + properties: + memory: + type: string + description: Maximum memory for the csi snapshot controller container. + cpu: + type: string + description: Maximum cpu for the csi snapshot controller container. + portworxApi: + type: object + description: Contains a spec to configure the Portworx API component. + properties: + resources: + type: object + description: Specifies the compute resource requirements for the portworx api container. + properties: + requests: + type: object + description: Requested resources for the portworx api container. + properties: + memory: + type: string + description: Requested memory for the portworx api container. + cpu: + type: string + description: Requested cpu for the portworx api container. + limits: + type: object + description: Limits describes the maximum amount of compute resources allowed for the portworx api container. + properties: + memory: + type: string + description: Maximum memory for the portworx api container. + cpu: + type: string + description: Maximum cpu for the portworx api container. env: type: array description: List of environment variables used by the driver. This is an array of Kubernetes diff --git a/deploy/olm-catalog/portworx/24.1.2-drw/core_v1_storagecluster_crd.yaml b/deploy/olm-catalog/portworx/24.1.2-drw/core_v1_storagecluster_crd.yaml new file mode 100644 index 0000000000..c804c178e5 --- /dev/null +++ b/deploy/olm-catalog/portworx/24.1.2-drw/core_v1_storagecluster_crd.yaml @@ -0,0 +1,3939 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: storageclusters.core.libopenstorage.org +spec: + group: core.libopenstorage.org + names: + kind: StorageCluster + listKind: StorageClusterList + plural: storageclusters + singular: storagecluster + shortNames: + - stc + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - description: The unique ID of the storage cluster + jsonPath: .status.clusterUid + name: Cluster UUID + type: string + - description: The status of the storage cluster + jsonPath: .status.phase + name: Status + type: string + - description: The version of the storage cluster + jsonPath: .spec.version + name: Version + type: string + - description: The age of the storage cluster + jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + description: The desired behavior of the storage cluster. + properties: + priorityClassName: + type: string + description: Priority class name that the operator will pass to the portworx storage pods to be scheduled accordingly. + metadata: + type: object + description: Metadata contains metadata for different storage cluster components. + properties: + annotations: + type: object + x-kubernetes-preserve-unknown-fields: true + description: >- + The annotations section of spec is a map of map to pass custom annotations to different + storage cluster components. The key specifies component in format of "kind/component", + e.g. "deployment/stork" to pass custom annotations to stork deployment. The value is a map of + string that contains custom annotation key and value pairs. + labels: + type: object + x-kubernetes-preserve-unknown-fields: true + description: >- + The labels section of spec is a map of map to pass custom labels to different storage cluster + components. The key specifies component in format of "kind/component", e.g. "service/portworx-api" + to pass custom labels to portworx-api service. The value is a map of string that contains custom + label key and value pairs. + resources: + type: object + description: Specifies the compute resource requirements for the storage pod. + properties: + requests: + type: object + description: Requested resources for the storage pod. + properties: + memory: + type: string + description: Requested memory for the storage pod. + cpu: + type: string + description: Requested cpu for the storage pod. + image: + type: string + description: Docker image of the storage driver. + version: + type: string + description: Version of the storage driver. This field is read-only. + imagePullPolicy: + type: string + description: Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always. + imagePullSecret: + type: string + description: Image pull secret is a reference to secret in the same namespace as the + StorageCluster. It is used for pulling all images used by the StorageCluster. + customImageRegistry: + type: string + description: >- + Custom container image registry server that will be used instead of + index.docker.io to download Docker images. This may include the repository as well. + (Example: myregistry.net:5443 or myregistry.com/myrepository) + preserveFullCustomImageRegistry: + type: boolean + description: >- + Setting this to true this stops part of the image tag being swallowed when setting a + customImageRegistry with a / in it. When set to false using a customImageRegistry of + `example.io/public` and an image of `portworx/oci-monitor` the full image path is is + `example.io/public/oci-monitor`, setting to true gives you + `example.io/public/portworx/oci-monitor`. Defaults to false + secretsProvider: + type: string + description: Secrets provider is the name of secret provider that driver will connect to. + startPort: + type: integer + format: int32 + minimum: 0 + description: Start port is the starting port in the range of ports used by the cluster. + autoUpdateComponents: + type: string + description: A strategy to determine how component versions are to be updated automatically. + updateStrategy: + type: object + description: An update strategy to replace existing StorageCluster pods with new pods. + properties: + type: + type: string + description: Type of storage cluster update. Can be RollingUpdate or OnDelete. + Default is RollingUpdate. + enum: + - RollingUpdate + - OnDelete + rollingUpdate: + type: object + description: Spec to control the desired behavior of storage cluster rolling update. + properties: + minReadySeconds: + description: Minimum number of seconds for which a newly created Portworx pod + should be ready without any of its container crashing for it to + be considered available. Defaults to 0 (pod will be considered available + as soon as it is ready). + format: int32 + type: integer + maxUnavailable: + x-kubernetes-int-or-string: true + description: >- + The maximum number of StorageCluster pods that can be unavailable + during the update. Value can be an absolute number (ex: 5) or a percentage of + total number of StorageCluster pods at the start of the update (ex: 10%). + Absolute number is calculated from percentage by rounding up. This cannot be 0. + Default value is 1. Example: when this is set to 30%, at most 30% of the total + number of nodes that should be running the storage pod can have their pods + stopped for an update at any given time. The update starts by stopping at most + 30% of those StorageCluster pods and then brings up new StorageCluster pods in + their place. Once the new pods are available, it then proceeds onto other + StorageCluster pods, thus ensuring that at least 70% of original number of + StorageCluster pods are available at all times during the update. + deleteStrategy: + type: object + description: Delete strategy to uninstall and wipe the storage cluster. + properties: + type: + type: string + description: Type of storage cluster delete. Can be Uninstall or UninstallAndWipe. + There is no default delete strategy. When no delete strategy only objects managed + by the StorageCluster controller and owned by the StorageCluster object are deleted. + The storage driver will be left in a state where it will not be managed by any object. + Uninstall strategy ensures that the cluster is completely uninstalled even from the + storage driver perspective. UninstallAndWipe strategy ensures that the cluster is + completely uninstalled as well as the storage devices and metadata are wiped for + reuse. This may result in data loss. + enum: + - Uninstall + - UninstallAndWipe + revisionHistoryLimit: + type: integer + format: int32 + description: The number of old history to retain to allow rollback. This is a pointer + to distinguish between an explicit zero and not specified. Defaults to 10. + featureGates: + type: object + x-kubernetes-preserve-unknown-fields: true + description: This is a map of feature names to string values. + runtimeOptions: + type: object + x-kubernetes-preserve-unknown-fields: true + description: This is map of any runtime options that need to be sent to the storage + driver. The value is a string. + placement: + type: object + description: Describes placement configuration for the storage cluster pods. + properties: + nodeAffinity: + type: object + description: Describes node affinity scheduling rules for the storage cluster pods. + This is exactly the same object as Kubernetes node affinity for pods. + properties: + requiredDuringSchedulingIgnoredDuringExecution: + type: object + properties: + nodeSelectorTerms: + type: array + items: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + required: + - nodeSelectorTerms + preferredDuringSchedulingIgnoredDuringExecution: + type: array + items: + type: object + properties: + weight: + type: integer + preference: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + required: + - preference + - weight + tolerations: + type: array + description: Tolerations for all the pods deployed by the StorageCluster controller. + The pod with this toleration attached will tolerate any taint that matches the + triple using the matching operator . + items: + type: object + properties: + effect: + type: string + description: Effect indicates the taint effect to match. Empty means match + all taint effects. When specified, allowed values are NoSchedule, + PreferNoSchedule and NoExecute. + key: + type: string + description: Key is the taint key that the toleration applies to. Empty means + match all taint keys. If the key is empty, operator must be Exists; this + combination means to match all values and all keys. + operator: + type: string + description: "Operator represents a key's relationship to the value. Valid + operators are Exists and Equal. Defaults to Equal. Exists is equivalent to + wildcard for value, so that a pod can tolerate all taints of a particular category." + value: + type: string + description: Value is the taint value the toleration matches to. If the operator + is Exists, the value should be empty, otherwise just a regular string. + tolerationSeconds: + type: integer + description: TolerationSeconds represents the period of time the toleration + (which must be of effect NoExecute, otherwise this field is ignored) tolerates + the taint. By default, it is not set, which means tolerate the taint forever + (do not evict). Zero and negative values will be treated as 0 (evict + immediately) by the system. + kvdb: + type: object + description: Details of KVDB that the storage driver will use. + properties: + internal: + type: boolean + description: Flag indicating whether to use internal KVDB or an external KVDB. + endpoints: + type: array + description: If using external KVDB, this is the list of KVDB endpoints. + items: + type: string + authSecret: + type: string + description: Authentication secret is the name of Kubernetes secret containing + information to authenticate with the external KVDB. It could have the username/password + for basic auth, certificate information or an ACL token. + storage: + type: object + description: Details of the storage used by the storage driver. + properties: + useAll: + type: boolean + description: Use all available, unformatted, unpartitioned devices. This will be + ignored if spec.storage.devices is not empty. + useAllWithPartitions: + type: boolean + description: Use all available unformatted devices. This will be + ignored if spec.storage.devices is not empty. + forceUseDisks: + type: boolean + description: Flag indicating to use the devices even if there is file system present + on it. Note that the devices may be wiped before using. + devices: + type: array + description: List of devices to be used by the storage driver. + items: + type: string + cacheDevices: + type: array + description: List of cache devices to be used by the storage driver. + items: + type: string + journalDevice: + type: string + description: Device used for journaling. + systemMetadataDevice: + type: string + description: Device that will be used to store system metadata by the driver. + kvdbDevice: + type: string + description: Device used for internal KVDB. + cloudStorage: + type: object + description: Details of storage used in cloud environment. + properties: + provider: + type: string + description: Cloud provider name. + maxStorageNodes: + type: integer + format: int32 + minimum: 0 + description: Maximum nodes that will have storage in the cluster. + maxStorageNodesPerZone: + type: integer + format: int32 + minimum: 0 + description: Maximum nodes in every zone that will have storage in the cluster. + maxStorageNodesPerZonePerNodeGroup: + type: integer + format: int32 + minimum: 0 + description: Maximum nodes in every zone in every node group that will have storage + in the cluster. + nodePoolLabel: + type: string + description: Kubernetes node label key with which nodes are grouped into node pools + for storage distribution in cloud environment. + deviceSpecs: + type: array + description: List of storage device specs. A cloud storage device will be created + for every spec in the list. The specs will be applied to all nodes in the cluster + up to spec.cloudStorage.maxStorageNodes or spec.cloudStorage.maxStorageNodesPerZone + or spec.cloudStorage.maxStorageNodesPerZonePerNodeGroup. + This will be ignored if spec.cloudStorage.capacitySpecs is present. + items: + type: string + capacitySpecs: + type: array + description: List of cluster wide storage types and their capacities. A single + capacity spec identifies a storage pool with a set of minimum requested IOPS + and size. Based on the cloud provider, the total storage capacity will get + divided amongst the nodes. The nodes bearing storage themselves will get + uniformly distributed across all the zones. + items: + type: object + properties: + minIOPS: + type: integer + format: int64 + minimum: 0 + description: Minimum IOPS expected from the cloud drive. + minCapacityInGiB: + type: integer + format: int64 + minimum: 0 + description: Minimum capacity for this storage cluster. The total capacity + of devices created by this capacity spec should not be less than this + number for the entire cluster. + maxCapacityInGiB: + type: integer + format: int64 + minimum: 0 + description: Maximum capacity for this storage cluster. The total capacity + of devices created by this capacity spec should not be greater than this + number for the entire cluster. + options: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Additional options required to provision the drive in cloud. + journalDeviceSpec: + type: string + description: Device spec for the journal device. + systemMetadataDeviceSpec: + type: string + description: Device spec for the metadata device. This device will be used to store + system metadata by the driver. + kvdbDeviceSpec: + type: string + description: Device spec for internal KVDB device. + network: + type: object + description: Contains network information that is needed by the storage driver. + properties: + dataInterface: + type: string + description: Name of the network interface used by the storage driver for data traffic. + mgmtInterface: + type: string + description: Name of the network interface used by the storage driver for management traffic. + stork: + type: object + description: Contains STORK related spec. + properties: + enabled: + type: boolean + description: Flag indicating whether STORK needs to be enabled. + lockImage: + type: boolean + description: Flag indicating if the STORK image needs to be locked to the given image. + If the image is not locked, it can be updated by the storage driver during upgrades. + image: + type: string + description: Docker image of the STORK container. + hostNetwork: + type: boolean + description: Flag indicating if Stork pods should run in host network. + args: + type: object + x-kubernetes-preserve-unknown-fields: true + description: >- + It is map of arguments given to STORK. Example: driver: pxd + env: + type: array + description: List of environment variables used by STORK. This is an array of + Kubernetes EnvVar where the value can be given directly or from a source like field, + config map or secret. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + volumes: + type: array + items: + type: object + properties: + name: + type: string + readOnly: + type: boolean + mountPath: + type: string + mountPropagation: + type: string + hostPath: + type: object + properties: + path: + type: string + type: + type: string + secret: + type: object + properties: + secretName: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + projected: + type: object + properties: + defaultMode: + type: integer + sources: + type: array + items: + type: object + properties: + secret: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + resources: + type: object + description: Specifies the resource requirements for stork and stork scheduler. + properties: + requests: + type: object + description: Requested resources. + properties: + memory: + type: string + description: Requested memory. + cpu: + type: string + description: Requested cpu. + limits: + type: object + description: Resource limit. + properties: + memory: + type: string + description: Memory limit. + cpu: + type: string + description: CPU limit. + userInterface: + type: object + description: Contains spec of a user interface for the storage driver. + properties: + enabled: + type: boolean + description: Flag indicating whether the user interface needs to be enabled. + lockImage: + type: boolean + description: Flag indicating if the user interface image needs to be locked to the given + image. If the image is not locked, it can be updated by the storage driver during upgrades. + image: + type: string + description: Docker image of the user interface container. + env: + type: array + description: List of environment variables used by the UI components. This is an array + of Kubernetes EnvVar where the value can be given directly or from a source like field, + config map or secret. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + autopilot: + type: object + description: Contains spec of autopilot component for storage driver. + properties: + enabled: + type: boolean + description: Flag indicating whether autopilot needs to be enabled. + lockImage: + type: boolean + description: Flag indicating if the autopilot image needs to be locked to the given image. + If the image is not locked, it can be updated by the storage driver during upgrades. + image: + type: string + description: Docker image of the autopilot container. + args: + type: object + x-kubernetes-preserve-unknown-fields: true + description: >- + It is a map of arguments provided to autopilot. Example: log-level: debug + env: + type: array + description: List of environment variables used by autopilot. This is an array of + Kubernetes EnvVar where the value can be given directly or from a source like field, + config map or secret. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + volumes: + type: array + items: + type: object + properties: + name: + type: string + readOnly: + type: boolean + mountPath: + type: string + mountPropagation: + type: string + hostPath: + type: object + properties: + path: + type: string + type: + type: string + secret: + type: object + properties: + secretName: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + projected: + type: object + properties: + defaultMode: + type: integer + sources: + type: array + items: + type: object + properties: + secret: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + providers: + type: array + description: List of input data providers to autopilot. + items: + type: object + properties: + name: + type: string + description: Unique name of the data provider. + type: + type: string + description: Type of the data provider. For instance - prometheus + params: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Map of key-value params for the provider. + resources: + type: object + description: Specifies the resource requirements for the autopilot pod. + properties: + requests: + type: object + description: Requested resources. + properties: + memory: + type: string + description: Requested memory. + cpu: + type: string + description: Requested cpu. + limits: + type: object + description: Resource limit. + properties: + memory: + type: string + description: Memory limit. + cpu: + type: string + description: CPU limit. + monitoring: + type: object + description: Contains monitoring configuration for the storage cluster. + properties: + enableMetrics: + type: boolean + description: "If this flag is enabled it will expose the storage cluster metrics to external + monitoring solutions like Prometheus. DEPRECATED - use prometheus.exportMetrics instead" + prometheus: + type: object + description: Contains configuration of Prometheus to monitor the storage cluster. + properties: + resources: + type: object + description: Define resources requests and limits for single Pods. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + securityContext: + description: 'Security options the pod should run with. More + info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by + the container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent to + root on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount to + use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root + filesystem. Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a + non-root user. If true, the Kubelet will validate the + image at runtime to ensure that it does not run as UID + 0 (root) and fail to start the container if it does. If + unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both + SecurityContext and PodSecurityContext, the value specified + in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata + if unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a + random SELinux context for each container. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set + in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + exportMetrics: + type: boolean + description: If this flag is enabled it will expose the storage cluster metrics to Prometheus. + enabled: + type: boolean + description: Flag indicating whether Prometheus stack needs to be enabled and deployed + by the Storage operator. + remoteWriteEndpoint: + type: string + description: Specifies the remote write endpoint for Prometheus. + alertManager: + type: object + description: Contains configuration of AlertManager for the storage cluster. + properties: + enabled: + type: boolean + description: Flag indicating whether AlertManager needs to be enabled and deployed + by the Storage operator. + replicas: + description: Total number of non-terminated pods targeted by this + Prometheus deployment (their labels match the selector). + format: int32 + type: integer + retention: + description: Time duration Prometheus shall retain data for. Default + is '24h' if retentionSize is not set, and must match the regular + expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes + hours days weeks years). + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + retentionSize: + description: Maximum amount of disk space used by blocks. + pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ + type: string + storage: + description: Storage spec to specify how storage shall be used. + properties: + disableMountSubPath: + description: 'Deprecated: subPath usage will be disabled by default + in a future release, this option will become unnecessary. DisableMountSubPath + allows to remove any subPath usage in volume mounts.' + type: boolean + emptyDir: + description: 'EmptyDirVolumeSource to be used by the StatefulSet. + If specified, used in place of any volumeClaimTemplate. More + info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir' + properties: + medium: + description: 'medium represents what type of storage medium + should back this directory. The default is "" which means + to use the node''s default medium. Must be an empty string + (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local storage + required for this EmptyDir volume. The size limit is also + applicable for memory medium. The maximum usage on memory + medium EmptyDir would be the minimum value between the SizeLimit + specified here and the sum of memory limits of all containers + in a pod. The default is nil which means that the limit + is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: 'EphemeralVolumeSource to be used by the StatefulSet. + This is a beta field in k8s 1.21, for lower versions, starting + with k8s 1.19, it requires enabling the GenericEphemeralVolume + feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes' + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC will + be deleted together with the pod. The name of the PVC will + be `-` where `` is the + name from the `PodSpec.Volumes` array entry. Pod validation + will reject the pod if the concatenated name is not valid + for a PVC (for example, too long). \n An existing PVC with + that name that is not owned by the pod will *not* be used + for the pod to avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the unrelated PVC + is removed. If such a pre-created PVC is meant to be used + by the pod, the PVC has to updated with an owner reference + to the pod once the pod exists. Normally this should not + be necessary, but it may be useful when manually reconstructing + a broken cluster. \n This field is read-only and no changes + will be made by Kubernetes to the PVC after it has been + created. \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations that will + be copied into the PVC when creating it. No other fields + are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified data + source. When the AnyVolumeDataSource feature gate + is enabled, dataSource contents will be copied to + dataSourceRef, and dataSourceRef contents will be + copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, + then dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object from + which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a + non-empty API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the dataSource field and as such if both fields + are non-empty, they must have the same value. For + backwards compatibility, when namespace isn''t specified + in dataSourceRef, both fields (dataSource and dataSourceRef) + will be set to the same value automatically if one + of them is empty and the other is non-empty. When + namespace is specified in dataSourceRef, dataSource + isn''t set to the same value and must be empty. + There are three important differences between dataSource + and dataSourceRef: * While dataSource only allows + two specific types of objects, dataSourceRef allows + any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed values + (dropping them), dataSourceRef preserves all values, + and generates an error if a disallowed value is + specified. * While dataSource only allows local + objects, dataSourceRef allows objects in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled. (Alpha) Using the namespace + field of dataSourceRef requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: Namespace is the namespace of resource + being referenced Note that when a namespace + is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent namespace + to allow that namespace's owner to accept the + reference. See the ReferenceGrant documentation + for details. (Alpha) This field requires the + CrossNamespaceVolumeDataSource feature gate + to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are used + by this container. \n This is an alpha field + and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable." + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of + one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes + that resource available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the + StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is + implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + volumeClaimTemplate: + description: A PVC spec to be used by the StatefulSet. The easiest + way to use a volume that cannot be automatically provisioned + (for whatever reason) is to use a label selector alongside manually + created PersistentVolumes. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: EmbeddedMetadata contains metadata relevant to + an EmbeddedResource. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value + map stored with a resource that may be set by external + tools to store and retrieve arbitrary metadata. They + are not queryable and should be preserved when modifying + objects. More info: http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be + used to organize and categorize (scope and select) objects. + May match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + name: + description: 'Name must be unique within a namespace. + Is required when creating resources, although some resources + may allow a client to request the generation of an appropriate + name automatically. Name is primarily intended for creation + idempotence and configuration definition. Cannot be + updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + type: object + spec: + description: 'Spec defines the desired characteristics of + a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on the + contents of the specified data source. When the AnyVolumeDataSource + feature gate is enabled, dataSource contents will be + copied to dataSourceRef, and dataSourceRef contents + will be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, then + dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object from + which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty + API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic provisioner. + This field will replace the functionality of the dataSource + field and as such if both fields are non-empty, they + must have the same value. For backwards compatibility, + when namespace isn''t specified in dataSourceRef, both + fields (dataSource and dataSourceRef) will be set to + the same value automatically if one of them is empty + and the other is non-empty. When namespace is specified + in dataSourceRef, dataSource isn''t set to the same + value and must be empty. There are three important differences + between dataSource and dataSourceRef: * While dataSource + only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed values + (dropping them), dataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + * While dataSource only allows local objects, dataSourceRef + allows objects in any namespaces. (Beta) Using this + field requires the AnyVolumeDataSource feature gate + to be enabled. (Alpha) Using the namespace field of + dataSourceRef requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource + being referenced Note that when a namespace is specified, + a gateway.networking.k8s.io/ReferenceGrant object + is required in the referent namespace to allow that + namespace's owner to accept the reference. See the + ReferenceGrant documentation for details. (Alpha) + This field requires the CrossNamespaceVolumeDataSource + feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the status + field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are used by + this container. \n This is an alpha field and requires + enabling the DynamicResourceAllocation feature gate. + \n This field is immutable." + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one + entry in pod.spec.resourceClaims of the Pod + where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes to + consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the StorageClass + required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is + required by the claim. Value of Filesystem is implied + when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the + PersistentVolume backing this claim. + type: string + type: object + status: + description: 'Status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'accessModes contains the actual access modes + the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: allocatedResources is the storage resource + within AllocatedResources tracks the capacity allocated + to a PVC. It may be larger than the actual capacity + when a volume expansion operation is requested. For + storage quota, the larger value from allocatedResources + and PVC.spec.resources is used. If allocatedResources + is not set, PVC.spec.resources alone is used for quota + calculation. If a volume expansion capacity request + is lowered, allocatedResources is only lowered if there + are no expansion operations in progress and if the actual + volume capacity is equal or lower than the requested + capacity. This is an alpha field and requires enabling + RecoverVolumeExpansionFailure feature. + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: capacity represents the actual resources + of the underlying volume. + type: object + conditions: + description: conditions is the current Condition of persistent + volume claim. If underlying persistent volume is being + resized then the Condition will be set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails + details about state of pvc + properties: + lastProbeTime: + description: lastProbeTime is the time we probed + the condition. + format: date-time + type: string + lastTransitionTime: + description: lastTransitionTime is the time the + condition transitioned from one status to another. + format: date-time + type: string + message: + description: message is the human-readable message + indicating details about last transition. + type: string + reason: + description: reason is a unique, this should be + a short, machine understandable string that gives + the reason for condition's last transition. If + it reports "ResizeStarted" that means the underlying + persistent volume is being resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType + is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: phase represents the current phase of PersistentVolumeClaim. + type: string + resizeStatus: + description: resizeStatus stores status of resize operation. + ResizeStatus is not set by default but when expansion + is complete resizeStatus is set to empty string by resize + controller or kubelet. This is an alpha field and requires + enabling RecoverVolumeExpansionFailure feature. + type: string + type: object + type: object + type: object + volumes: + description: Volumes allows configuration of additional volumes on + the output StatefulSet definition. Volumes specified will be appended + to other volumes that are generated as a result of StorageSpec objects. + items: + description: Volume represents a named volume in a pod that may + be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'awsElasticBlockStore represents an AWS Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume that + you want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force the readOnly + setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the persistent disk + resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount on + the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the + blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + description: fsType is Filesystem type to mount. Must be + a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host that + shares a pod's lifetime + properties: + monitors: + description: 'monitors is Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'readOnly is Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile is the + path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef is reference + to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is optional: User is the rados user name, + default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'cinder represents a cinder volume attached and + mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to + be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points to a secret + object containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: 'volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate + this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode bits used to + set permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults to + 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value pair in + the Data field of the referenced ConfigMap will be projected + into the volume as a file whose name is the key and content + is the value. If specified, the listed keys will be projected + into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in + the ConfigMap, the volume setup will error unless it is + marked optional. Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. If not + specified, the volume defaultMode will be used. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral + storage that is handled by certain external CSI drivers (Beta + feature). + properties: + driver: + description: driver is the name of the CSI driver that handles + this volume. Consult with your admin for the correct name + as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated + CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference to the + secret object containing sensitive information to pass + to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the secret + object contains more than one secret, all secret references + are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: volumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod + that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a Optional: mode bits used to set + permissions on created files by default. Must be an octal + value between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults to + 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name and namespace are + supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used to set permissions + on this file, must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, + the volume defaultMode will be used. This might + be in conflict with other options that affect the + file mode, like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary directory that + shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type of storage medium + should back this directory. The default is "" which means + to use the node''s default medium. Must be an empty string + (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local storage + required for this EmptyDir volume. The size limit is also + applicable for memory medium. The maximum usage on memory + medium EmptyDir would be the minimum value between the + SizeLimit specified here and the sum of memory limits + of all containers in a pod. The default is nil which means + that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is tied + to the pod that defines it - it will be created before the + pod starts, and deleted when the pod is removed. \n Use this + if: a) the volume is only needed while the pod runs, b) features + of normal volumes like restoring from snapshot or capacity + tracking are needed, c) the storage driver is specified through + a storage class, and d) the storage driver supports dynamic + volume provisioning through a PersistentVolumeClaim (see EphemeralVolumeSource + for more information on the connection between this volume + type and PersistentVolumeClaim). \n Use PersistentVolumeClaim + or one of the vendor-specific APIs for volumes that persist + for longer than the lifecycle of an individual pod. \n Use + CSI for light-weight local ephemeral volumes if the CSI driver + is meant to be used that way - see the documentation of the + driver for more information. \n A pod can use both types of + ephemeral volumes and persistent volumes at the same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The name of the + PVC will be `-` where `` is the name from the `PodSpec.Volumes` array entry. + Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too long). \n + An existing PVC with that name that is not owned by the + pod will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC + is meant to be used by the pod, the PVC has to updated + with an owner reference to the pod once the pod exists. + Normally this should not be necessary, but it may be useful + when manually reconstructing a broken cluster. \n This + field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. \n Required, must + not be nil." + properties: + metadata: + description: May contain labels and annotations that + will be copied into the PVC when creating it. No other + fields are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified + data source. When the AnyVolumeDataSource feature + gate is enabled, dataSource contents will be copied + to dataSourceRef, and dataSourceRef contents will + be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, + then dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object + from which to populate the volume with data, if + a non-empty volume is desired. This may be any + object from a non-empty API group (non core object) + or a PersistentVolumeClaim object. When this field + is specified, volume binding will only succeed + if the type of the specified object matches some + installed volume populator or dynamic provisioner. + This field will replace the functionality of the + dataSource field and as such if both fields are + non-empty, they must have the same value. For + backwards compatibility, when namespace isn''t + specified in dataSourceRef, both fields (dataSource + and dataSourceRef) will be set to the same value + automatically if one of them is empty and the + other is non-empty. When namespace is specified + in dataSourceRef, dataSource isn''t set to the + same value and must be empty. There are three + important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types + of objects, dataSourceRef allows any non-core + object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping + them), dataSourceRef preserves all values, and + generates an error if a disallowed value is specified. + * While dataSource only allows local objects, + dataSourceRef allows objects in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled. (Alpha) Using the + namespace field of dataSourceRef requires the + CrossNamespaceVolumeDataSource feature gate to + be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: Namespace is the namespace of resource + being referenced Note that when a namespace + is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent namespace + to allow that namespace's owner to accept + the reference. See the ReferenceGrant documentation + for details. (Alpha) This field requires the + CrossNamespaceVolumeDataSource feature gate + to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are used + by this container. \n This is an alpha field + and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable." + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available inside + a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If Requests + is omitted for a container, it defaults to + Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the + StorageClass required by the claim. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem + is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to the + pod. + properties: + fsType: + description: 'fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. TODO: how do we prevent errors in the + filesystem from compromising the machine' + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs and + lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: flexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends + on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef is reference + to the secret object containing sensitive information + to pass to the plugin scripts. This may be empty if no + secret object is specified. If the secret object contains + more than one secret, all secrets are passed to the plugin + scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached to + a kubelet's host machine. This depends on the Flocker control + service being running + properties: + datasetName: + description: datasetName is Name of the dataset stored as + metadata -> name on the dataset for Flocker should be + considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume that + you want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the PD resource in + GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the EmptyDir into + the Pod''s container.' + properties: + directory: + description: directory is the target directory name. Must + not contain or start with '..'. If '.' is supplied, the + volume directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume path. More info: + https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the Glusterfs volume + to be mounted with read-only permissions. Defaults to + false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing file or directory + on the host machine that is directly exposed to the container. + This is generally used for system agents or other privileged + things that are allowed to see the host machine. Most containers + will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host directory + mounts and who can/can not mount host directories as read/write.' + properties: + path: + description: 'path of the directory on the host. If the + path is a symlink, it will follow the link to the real + path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults to "" More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource that is + attached to a kubelet''s host machine and then exposed to + the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI + Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI Initiator + Name. If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface Name that uses + an iSCSI transport. Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal List. The + portal is either an IP or ip_addr:port if the port is + other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: targetPortal is iSCSI Target Portal. The Portal + is either an IP or ip_addr:port if the port is other than + default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL and unique + within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'nfs represents an NFS mount on the host that shares + a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the NFS server. More + info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the NFS export to + be mounted with read-only permissions. Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP address of the + NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource represents a + reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: readOnly Will force the ReadOnly setting in + VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits used to set permissions + on created files by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires decimal + values for mode bits. Directories within the path are + not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along with + other supported volume types + properties: + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced ConfigMap + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified which + is not present in the ConfigMap, the volume + setup will error unless it is marked optional. + Paths must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 and + 0777 or a decimal value between 0 and + 511. YAML accepts both octal and decimal + values, JSON requires decimal values for + mode bits. If not specified, the volume + defaultMode will be used. This might be + in conflict with other options that affect + the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of + the file to map the key to. May not be + an absolute path. May not contain the + path element '..'. May not start with + the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used to + set permissions on this file, must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' + path. Must be utf-8 encoded. The first + item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the + container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu + and requests.memory) are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the secret data + to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced Secret + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified which + is not present in the Secret, the volume setup + will error unless it is marked optional. Paths + must be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 and + 0777 or a decimal value between 0 and + 511. YAML accepts both octal and decimal + values, JSON requires decimal values for + mode bits. If not specified, the volume + defaultMode will be used. This might be + in conflict with other options that affect + the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of + the file to map the key to. May not be + an absolute path. May not contain the + path element '..'. May not start with + the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: optional field specify whether the + Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about + the serviceAccountToken data to project + properties: + audience: + description: audience is the intended audience + of the token. A recipient of a token must identify + itself with an identifier specified in the audience + of the token, and otherwise should reject the + token. The audience defaults to the identifier + of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, the + kubelet volume plugin will proactively rotate + the service account token. The kubelet will + start trying to rotate the token if the token + is older than 80 percent of its time to live + or if the token is older than 24 hours.Defaults + to 1 hour and must be at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative to the + mount point of the file to project the token + into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: group to map volume access to Default is no + group + type: string + readOnly: + description: readOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults to + false. + type: boolean + registry: + description: registry represents a single or multiple Quobyte + Registry services specified as a string as host:port pair + (multiple entries are separated with commas) which acts + as the central registry for volumes + type: string + tenant: + description: tenant owning the given Quobyte volume in the + Backend Used with dynamically provisioned Quobyte volumes, + value is set by the plugin + type: string + user: + description: user to map volume access to Defaults to serivceaccount + user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'image is the rados image name. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication secret + for RBDUser. If provided overrides keyring. Default is + nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is the rados user name. Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret for ScaleIO + user and other sensitive information. If this is not provided, + Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: storageMode indicates whether the storage for + a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system as + configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume already + created in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode bits used to + set permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults to + 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value pair in + the Data field of the referenced Secret will be projected + into the volume as a file whose name is the key and content + is the value. If specified, the listed keys will be projected + into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in + the Secret, the volume setup will error unless it is marked + optional. Paths must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. If not + specified, the volume defaultMode will be used. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether the Secret or + its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the secret in the + pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret to use for obtaining + the StorageOS API credentials. If not specified, default + values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: volumeName is the human-readable name of the + StorageOS volume. Volume names are only unique within + a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the scope of the + volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS + for tighter integration. Set VolumeName to any name to + override the default behaviour. Set to "default" if you + are not using namespaces within StorageOS. Namespaces + that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fsType is filesystem type to mount. Must be + a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + volumeMounts: + description: VolumeMounts allows configuration of additional VolumeMounts + on the output StatefulSet definition. VolumeMounts specified will + be appended to other VolumeMounts in the prometheus container, that + are generated as a result of StorageSpec objects. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume should + be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are propagated + from the host to container and the other way around. When + not set, MountPropagationNone is used. This field is beta + in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which the + container's volume should be mounted. Behaves similarly to + SubPath but environment variable references $(VAR_NAME) are + expanded using the container's environment. Defaults to "" + (volume's root). SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + grafana: + type: object + description: Contains configuration of Grafana to monitor the storage cluster. + properties: + enabled: + type: boolean + description: Flag indicating whether Grafana stack needs to be enabled and deployed + by the Storage operator. + telemetry: + type: object + description: Contains telemetry configuration for the storage cluster. + properties: + enabled: + type: boolean + description: Flag indicates if telemetry component needs to be enabled + image: + type: string + description: Docker image of the telemetry container. + logUploaderImage: + type: string + description: Docker image of the log-upload-service container. + security: + type: object + description: Contains security configuration for the storage cluster. + properties: + enabled: + type: boolean + description: Flag indicating whether security features need to be enabled for the storage cluster. + auth: + type: object + description: Authorization configurations for a RBAC enabled storage cluster + properties: + guestAccess: + type: string + description: Defines the access mode of a guest user + selfSigned: + type: object + description: Configuration for self signed authentication + properties: + issuer: + type: string + description: Token issuer for the tokens used to connect with storage cluster. + tokenLifetime: + type: string + description: Lifetime of auto-generated RBAC tokens to access the storage cluster. + sharedSecret: + type: string + description: Shared secret is the name of the Kubernetes secret containing the shared key + used for signing RBAC tokens. The secret has to be present in the StorageCluster namespace. + csi: + type: object + description: Contains CSI configuration for the storage cluster. + properties: + enabled: + type: boolean + description: Flag indicating whether CSI needs to be installed for the storage cluster. + installSnapshotController: + type: boolean + description: Flag indicating whether CSI Snapshot Controller needs to be installed for the storage cluster. + topology: + type: object + description: Contains CSI topology configurations. + properties: + enabled: + type: boolean + description: Flag indicating whether CSI topology feature gate is enabled. + env: + type: array + description: List of environment variables used by the driver. This is an array of Kubernetes + EnvVar where the value can be given directly or from a source like field, config map or secret. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + volumes: + type: array + items: + type: object + properties: + name: + type: string + readOnly: + type: boolean + mountPath: + type: string + mountPropagation: + type: string + hostPath: + type: object + properties: + path: + type: string + type: + type: string + secret: + type: object + properties: + secretName: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + projected: + type: object + properties: + defaultMode: + type: integer + sources: + type: array + items: + type: object + properties: + secret: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + nodes: + type: array + description: Node level configurations that will override the configuration at cluster level. + These configurations can be for individual nodes or can be grouped to override configuration + of multiple nodes based on label selectors. + items: + type: object + properties: + selector: + type: object + description: Configuration in this node block is applied to nodes based on this selector. + Use either nodeName of labelSelector, not both. If nodeName is used, labelSelector will + be ignored. + properties: + nodeName: + type: string + description: Name of the Kubernetes node that is to be selected. If present then the + labelSelector is ignored even if the node with the given name is absent and the + labelSelector matches another node. + labelSelector: + type: object + description: It is a label query over all the nodes. The result of matchLabels and + matchExpressions is ANDed. An empty label selector matches all nodes. A null + label selector matches no objects. + properties: + matchLabels: + type: object + x-kubernetes-preserve-unknown-fields: true + description: It is a map of key-value pairs. A single key-value in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + matchExpressions: + type: array + description: It is a list of label selector requirements. The requirements are ANDed. + items: + type: object + properties: + key: + type: string + description: It is the label key that the selector applies to. + operator: + type: string + description: "It represents a key's relationship to a set of values. Valid operators + are In, NotIn, Exists and DoesNotExist." + values: + type: array + description: It is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + items: + type: string + storage: + type: object + description: Details of the storage used by the storage driver. + properties: + useAll: + type: boolean + description: Use all available, unformatted, unpartitioned devices. This will be + ignored if spec.storage.devices is not empty. + useAllWithPartitions: + type: boolean + description: Use all available unformatted devices. This will be + ignored if spec.storage.devices is not empty. + forceUseDisks: + type: boolean + description: Flag indicating to use the devices even if there is file system present + on it. Note that the devices may be wiped before using. + devices: + type: array + description: List of devices to be used by the storage driver. + items: + type: string + cacheDevices: + type: array + description: List of cache devices to be used by the storage driver. + items: + type: string + journalDevice: + type: string + description: Device used for journaling. + systemMetadataDevice: + type: string + description: Device that will be used to store system metadata by the driver. + kvdbDevice: + type: string + description: Device used for internal KVDB. + cloudStorage: + type: object + description: Details of storage used in cloud environment. + properties: + deviceSpecs: + type: array + description: List of storage device specs. A cloud storage device will be created + for every spec in the list. The specs will be applied to all nodes in the cluster + that match the node group selector. The number of nodes that will come up with + storage are constrained by maxStorageNodes, maxStorageNodesPerZone and + maxStorageNodesPerZonePerNodeGroup. + items: + type: string + journalDeviceSpec: + type: string + description: Device spec for the journal device. + systemMetadataDeviceSpec: + type: string + description: Device spec for the metadata device. This device will be used to store + system metadata by the driver. + kvdbDeviceSpec: + type: string + description: Device spec for internal KVDB device. + maxStorageNodesPerZonePerNodeGroup: + type: integer + format: int32 + minimum: 0 + description: Maximum nodes in every zone in every node group that will have storage + in the cluster. + network: + type: object + description: Contains network information that is needed by the storage driver. + properties: + dataInterface: + type: string + description: Name of the network interface used by the storage driver for data traffic. + mgmtInterface: + type: string + description: Name of the network interface used by the storage driver for + management traffic. + runtimeOptions: + type: object + x-kubernetes-preserve-unknown-fields: true + description: This is map of any runtime options that need to be sent to the storage + driver. The value is a string. If runtime options are present here at node level, + they will override the ones from cluster configuration. + env: + type: array + description: List of environment variables used by the driver. This is an array + of Kubernetes EnvVar where the value can be given directly or from a source + like field, config map or secret. Environment variables specified here at the + node level will be merged with the ones present in cluster configuration and + sent to the nodes. If there is duplicate, the node level value will take precedence. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + status: + type: object + description: Most recently observed status of the storage cluster. This data may not be up to date. + properties: + clusterName: + type: string + description: Name of the storage cluster. + version: + type: string + description: Version of the storage driver. + clusterUid: + type: string + description: Unique ID of the storage cluster. + phase: + type: string + description: Phase of the StorageCluster is a simple, high-level summary of where the + StorageCluster is in its lifecycle. The condition array contains more detailed + information about the state of the cluster. + reason: + type: string + description: CamelCase messages split with commas indicating details about why the StorageCluster is in this state. + collisionCount: + type: integer + format: int32 + description: Count of hash collisions for the StorageCluster. The StorageCluster controller + uses this field as a collision avoidance mechanism when it needs to create the name of + the newest ControllerRevision. + storage: + type: object + description: Contains details of storage in the cluster. + properties: + storageNodesPerZone: + type: integer + format: int64 + description: The number of storage nodes per zone in the cluster. + desiredImages: + type: object + description: Represents all the desired images of various components. + properties: + stork: + type: string + description: Desired image for stork. + userInterface: + type: string + description: Desired image for user interface. + autopilot: + type: string + description: Desired image for autopilot. + csiNodeDriverRegistrar: + type: string + description: Desired image for CSI node driver registrar. + csiDriverRegistrar: + type: string + description: Desired image for CSI driver registrar. + csiProvisioner: + type: string + description: Desired image for CSI provisioner. + csiAttacher: + type: string + description: Desired image for CSI attacher. + csiResizer: + type: string + description: Desired image for CSI resizer. + csiSnapshotter: + type: string + description: Desired image for CSI snapshotter. + csiSnapshotController: + type: string + description: Desired image for CSI snapshot controller. + csiHealthMonitorController: + type: string + description: Desired image for CSI health monitor controller. + prometheusOperator: + type: string + description: Desired image for Prometheus operator. + prometheusConfigMapReload: + type: string + description: Desired image for Prometheus config map reload. + prometheusConfigReloader: + type: string + description: Desired image for Prometheus config reloader. + prometheus: + type: string + description: Desired image for Prometheus. + grafana: + type: string + description: Desired image for Grafana. + alertManager: + type: string + description: Desired image for AlertManager. + telemetry: + type: string + description: Desired image for telemetry. + metricsCollector: + type: string + description: Desired image for metrics collector. + metricsCollectorProxy: + type: string + description: Desired image for metrics collector proxy. + telemetryProxy: + type: string + description: Desired image for telemetry proxy. + logUploader: + type: string + description: Desired image for log uploader. + kubeScheduler: + type: string + description: Desired image for kubernetes scheduler. + kubeControllerManager: + type: string + description: Desired image for kubernetes controller manager. + pause: + type: string + description: Desired image for pause image. + dynamicPlugin: + type: string + description: Desired image for dynamic plugin image. + dynamicPluginProxy: + type: string + description: Desired image for nginx proxy image. + conditions: + type: array + description: Contains details for the current condition of this cluster. + items: + type: object + properties: + source: + type: string + description: Name of the component. + type: + type: string + description: Type of the condition. + status: + type: string + description: Status of the condition. + message: + type: string + description: Message is human readable message indicating details about the current + state of the cluster. + lastTransitionTime: + type: string + format: date-time + description: Time at which the condition changed. + - name: v1alpha1 + served: false + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/deploy/olm-catalog/portworx/24.1.2-drw/core_v1_storagenode_crd.yaml b/deploy/olm-catalog/portworx/24.1.2-drw/core_v1_storagenode_crd.yaml new file mode 100644 index 0000000000..829969abfd --- /dev/null +++ b/deploy/olm-catalog/portworx/24.1.2-drw/core_v1_storagenode_crd.yaml @@ -0,0 +1,191 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: storagenodes.core.libopenstorage.org +spec: + group: core.libopenstorage.org + names: + kind: StorageNode + listKind: StorageNodeList + plural: storagenodes + singular: storagenode + shortNames: + - sn + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: ID + type: string + description: The corresponding Kubernetes node name for the storage node + jsonPath: .status.nodeUid + - name: Status + type: string + description: The status of the storage node + jsonPath: .status.phase + - name: Version + type: string + description: The version of the storage node + jsonPath: .spec.version + - name: Age + type: date + description: The age of the storage cluster + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + description: The desired behavior of the storage node. Currently changing the spec does + not affect the actual storage node in the cluster. Eventually spec in StorageNode will + override the spec from StorageCluster so that configuration can be overridden at node + level. + properties: + version: + type: string + description: Version of the storage driver on the node. + cloudStorage: + type: object + description: Details of storage on the node for cloud environments. + properties: + driveConfigs: + type: array + description: List of cloud drive configs for the storage node. + items: + type: object + properties: + type: + type: string + description: Type of cloud drive. + sizeInGiB: + type: integer + format: int64 + minimum: 0 + description: Size of cloud drive in GiB. + iops: + type: integer + format: int64 + minimum: 0 + description: IOPS required from the cloud drive. + options: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Additional options for the cloud drive. + status: + type: object + description: Most recently observed status of the storage node. The data may not be up + to date. + properties: + nodeUid: + type: string + description: Unique ID of the storage node. + phase: + type: string + description: Phase of the StorageNode is a simple, high-level summary of where + the StorageNode is in its lifecycle. The condition array contains more detailed + information about the state of the node. + network: + type: object + description: Contains network information used by the storage node + properties: + dataIP: + type: string + description: IP address used by the storage driver for data traffic. + mgmtIP: + type: string + description: IP address used by the storage driver for management traffic. + storage: + type: object + description: Contains details of the status of storage for the node + properties: + totalSize: + type: string + description: Cumulative total size of all storage pools on the node. + usedSize: + type: string + description: Cumulative used size of all storage pools on the node. + conditions: + type: array + description: Contains details for the current condition of this storage node. + items: + type: object + properties: + type: + type: string + description: Type of the condition. + status: + type: string + description: Status of the condition. + reason: + type: string + description: Reason is a unique one-word reason about the current state + of the cluster. + message: + type: string + description: Message is the human readable message indicating details about the + current state of the cluster. + lastTransitionTime: + type: string + format: date-time + description: Time at which the condition changed. + checks: + type: array + description: Contains list of pre or post flight checks that are performed by the Operator + items: + type: object + properties: + type: + type: string + description: Type of the check. + reason: + type: string + description: Reason for success or failure of the check + success: + type: boolean + description: If true, the check was successful + result: + type: string + description: Result of the check fatal, warning, success + geography: + type: object + description: Contains topology information for the storage node. + properties: + region: + type: string + description: Region in which the storage node is placed. + zone: + type: string + description: Zone in which the storage node is placed. + rack: + type: string + description: Rack on which the storage node is placed. + operatingSystem: + type: string + description: Operating system of the underlying host. + kernelVersion: + type: string + description: Kernel version of the underlying host. + nodeAttributes: + type: object + description: Attributes of the storage node. + properties: + storage: + type: boolean + description: Indicates whether the node is a storage node or not. + kvdb: + type: boolean + description: Indicates whether the node is a kvdb node or not. + - name: v1alpha1 + served: false + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/deploy/olm-catalog/portworx/24.1.2-drw/portworx-certified.clusterserviceversion.yaml b/deploy/olm-catalog/portworx/24.1.2-drw/portworx-certified.clusterserviceversion.yaml new file mode 100644 index 0000000000..8f97bf29ae --- /dev/null +++ b/deploy/olm-catalog/portworx/24.1.2-drw/portworx-certified.clusterserviceversion.yaml @@ -0,0 +1,551 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: portworx-operator.v24.1.2-drw + namespace: placeholder + annotations: + capabilities: Auto Pilot + categories: "Storage" + description: Cloud native storage solution for production workloads + containerImage: portworx/px-operator:24.1.2-drw-dev + repository: https://github.com/libopenstorage/operator + createdAt: 2024-07-25T08:58:00Z + support: Portworx, Inc + certified: "true" + console.openshift.io/plugins: '["portworx"]' + features.operators.openshift.io/disconnected: "true" + features.operators.openshift.io/fips-compliant: "false" + features.operators.openshift.io/proxy-aware: "false" + features.operators.openshift.io/tls-profiles: "false" + features.operators.openshift.io/token-auth-aws: "false" + features.operators.openshift.io/token-auth-azure: "false" + features.operators.openshift.io/token-auth-gcp: "false" + operatorframework.io/initialization-resource: |- + { + "apiVersion": "core.libopenstorage.org/v1", + "kind": "StorageCluster", + "metadata": { + "name": "portworx", + "annotations": { + "portworx.io/is-openshift": "true" + } + } + } + alm-examples: |- + [ + { + "apiVersion": "core.libopenstorage.org/v1", + "kind": "StorageCluster", + "metadata": { + "name": "portworx", + "namespace": "test-operator", + "annotations": { + "portworx.io/is-openshift": "true" + } + }, + "spec": {} + }, + { + "apiVersion": "core.libopenstorage.org/v1", + "kind": "StorageNode", + "metadata": { + "name": "example", + "namespace": "test-operator" + }, + "spec": {} + } + ] +spec: + displayName: Portworx Enterprise + version: 24.1.2-drw + minKubeVersion: "1.21.0" + maturity: stable + replaces: portworx-operator.v24.1.1 + maintainers: + - name: Portworx + email: support@portworx.com + provider: + name: Portworx by PureStorage + keywords: ["portworx", "persistent storage", "storage", "cloud native", "open source"] + relatedImages: + - name: portworx-operator + image: portworx/px-operator:24.1.2-drw-dev + labels: + operated-by: portworx-operator + selector: + matchLabels: + operated-by: portworx-operator + links: + - name: Product Features + url: https://portworx.com/products/features + - name: Documentation + url: https://docs.portworx.com/portworx-install-with-kubernetes/on-premise/openshift/operator + - name: Support + url: https://docs.portworx.com/portworx-enterprise/support/contact-support + - name: Source Code + url: https://github.com/libopenstorage/operator + icon: + - base64data: iVBORw0KGgoAAAANSUhEUgAAAoAAAAD1CAYAAADAvUR4AACAAElEQVR4nOy9CZwkRZn//TwRmdV390xPX1UNzIAHorhyiawil4jKta684KrrzYrCuoKIrqDCKIL3rbDruf9VVFBZGeRUBAQFD0BRQUXm7Ko+Zqbp6bOqMuJ5P5EV0R1dU1XdPdNH9fTz5VNUT2ZWZkRmZMQvnifiCQEMwzAMwzDMioIFIMMwDMMwzAqDBSDDMAzDMMwKgwUgwzAMwzDMCoMFIMMwDMMwzAqDBSDDMAzDMMwKgwUgwzAMwzDMCoMFIMMwDMMwzAqDBSDDMAzDMMwKgwUgwzAMwzDMCoMFIMMwDMMwzAqDBSDDMAzDMMwKgwUgwzAMwzDMCoMFIMMwDMMwzAqDBSDDMAzDMMwKgwUgwzAMwzDMCoMFIMMwDMMwzAqDBSDDMAzDMMwKgwUgwzAMwzDMCoMFIMMwDMMwzAqDBSDDMAzDMMwKgwUgwzAMwzDMCoMFIMMwDMMwzAqDBSDDMAzDMMwKgwUgwzAMwzDMCiNY6gQsVwgA4WwroG8w/4w/DrMP3T4E0EuUTIZhGIZhGGZvMKKPAOQe/lYQW1wZhmEYhqkC2AI4C2JrHwBaS56Kt52eeJYmcSQhPY9AH6gBO5BIAmpCxGEk2oqC/qyVfmhwBB7Ge2DEO59gqyDDMAzDMEsFLnUCqh1frI2fWrs2QHgdEJ6lBTwvEYAE5/01d1JbG6GgqV9rAiLaCkB3Kg3/L9wA98DUL8z/aCnzxzAMwzDMyoMFYBl84bf95LpUcwLfBwhvCkNshohgnGKxp1AToYjlHGJhm/kDUJtTxP8WgUQB0vwz3vQzFcFVwQa4K77O2SDxhoJVkWEYhmEYZjFgAViE7+69HkCeeVrD+QLwslBi50SkQQNFQCQQCc1B6H4lPAEIRJPbgewO0gggRFgYB0gE/w8VXI4/hk3AbmGGYRiGYRYRFoAevggbOaXhpUEgrqoJ8KisIoiUjhBJxkJPw6TVzyg5xIJsnGYBJJicNhK7eeNtBCiMhgTEWhSkaJAIPi5+D5/DJyDLbmGGYRiGYRYDFoBFwm/oxOan1dTieingdQIJxiIdAYLE2CpIhPEdIywIvYIpb0YBWNgX+39RGNEYmwiVkBhAAoEUPIqgL8PrYQOwW5hhGIZhmAVmRQtA3+K29Rioa1u96iIEem9NgC0jOa2pIN5EbOaL/cJG+PlCb/YCMBZ/QJPWPRJmKxEJVDLEoCAv6Uf5rP5AzY3wGLBbmGEYhmGYBWLFCkCC2KoXW9mGX9ryz0GAV9ZK8ezRSIMmHaHZL2KJZq12pYTeHghAe8cJ7REy1pgaBJFICEkaRoHosziqPoE3wXBRCBqGYRiGYZi9ZsUFJr7eBnI24q/v5NXPHX15649rw+BHiPjsXTkdqVjxOYscFMb5LQxYGEOIRgsamSiiHCmtoQET4gO6OXhI/Wv4GoTCSiJ0OQhawYKdYRiGYZj5Y8UIissBxHprRRs8vmVVmAj/U0h6V63A2qFI6dhqBwWbX+FvwIIZr5Klb68sgAXfs0DAwjRhAGF+aa6PSoYigACAFNweRXRZ4rr878wh158N8hweH8gwDMMwzF6wzwvAYhfqrpPXvF5KXF8v8cAhpUETKYkkCoGcjQgz0ozs4MBFE4BA9oLmGwUiYBxWmoIaITVBHgG+jNnsR/G7sB14fCDDMAzDMHvBPi0Ap83uPan1aBHIq+sknjRhFJXWESDKghQjioUfgYvjtzQC0H7HpkeBdq4xKBAow1oEHVEPAFwhv5n9mstfYUoyh41hGIZhGGb27JMC0Bd+/ce3d9Ul8AOA+PaEADmitYqX6xUgYNrsXivaqkEAxvsKbuF4tnAcdZpUIEUACQAdwX1a0aXhtyZ+ARw2hmEYhmGYObJPTQJxK/Ea8Xc5gBg8qfOCuhr5UF0gL8hpECNKKwCUGCuuWI1hFYtgtBNQ7JJyGOSJdD4LkZB4LIby3uhtdV8de13dfk780eX71vNkGIZhGGZhqFbxM2emrd17YudJgRRX1wd49JjSEFG8fJuMw7kUlmlDZ7WbsuBVnQWw4Ns1fwu0CUcXP1AhChHWIeqItmuEq4LHRr+I90DEq4kwDMMwDDMTy95iZMfBGcGje4/rPPCpk7v/JxEEPwsFHj2U11Fek3ZhXayaW56iF23KC2FqpMlKboIiImwLQvEZ9ZzGByfOq3+5DRtDdHYh3A3DMAzDMEwxy1MMFa3i8benQ83qA7ovkgLeVy/EqqEo0ihiJ68Eb/k231K37CyA9m+KhwQWsk8iXpbYfKlEAgNznCL6ntS5D+JXsk+AdQvjep4tzDAMwzDMFMtSAF4PIM+xq3hsP3H/M4WAKxsCfO6wXcUDgGQcSoU0eZYz2FcEINhjyKpbtG5howRraoVQmoYJ4JMDT+76dOpmGGO3MMMwDMMwPstKAPrCb8uL9n9OfQ1+pEbgPysgmNAqQgIZR9ArhNMrREgpI9T2CQFY+LtwcQFIGJsDlUAMEnUCVESPKaIP1nx+6IfAs4UZhmEYhrEsCwHor+Lxt6Nbm1c3NL5PoLioRkLdrnxhFQ8QIJDIxVOetPrt6wJw6vipSSKFuIaoakIRgESINN1Mii5LfP6pPwC7hRmGYRhmxVPtAjCWVm52b98JB75OClpfL8XThiIFhKRQg0BBU7N7Ybo4W0ECMD4XFQY7FpaVs6uJ1NVKqYgmNMEXx8bU1auuGRoEXk2EYRiGYVYs1T4L2OgbvfW4A58/cMJBd9RJ8W0AfNpTeRVRbO1DWRBssbqqdjG7OGDBCQ5U0IaEIMezWuUjqA0T4pL6xuDh8Ytb3wh25rSNHcj3jmEYhmFWEFXd8G89Zr/umtrExYDwzhoUwYiKFBSMYaIw/8Eu5OFZ8Fa8BbAwS7hgN0UkDXZtYZNSASoRiECGCHkFP49IXVr/iZ0PAI8PZBiGYZgVRbVaAPHJF3R0ikTNLe21iYvyGsRwpHKAKK3UKVj8lmtMv0WC3Goi5Gb/YpBTpMeyWoUBnhgIeX/2/R1fGb2kvotXE2EYhmGYlUO1NvZ04IP9/bls9E9947lrQoHYGIgEECm3Llosaog4rMlscGK5IATN37FbOFIgEgl8RxA2PTJxafsFl0NhcggBCKpy6zDDMAzDMHvOsmjke4496EVhAFfVB3jcuNaQ19Nj/YnYPUrILuDpLmAqcT5EJI3OLYxEglQgRZCoEZCP6MGc0pc1XtX/M2C3MMMwDMPss1S1ALQSbXIWcO8Ja98qEa9oCMR+Q5Ey+kWBBikEERVEG88CnkEAxt/CDZyMfxPPFkZEXV8jgggANMH/jk3oD63+eN8m4LAxDMMwDLPPUdUC0OGHK3ns+ak17Q3BZSjw3xMSw5FIqVjacBzAuQpAJ5d3W02koU6KnKJBAPr4lu2Zzz3ji5Dl1UQYhmEYZt+hWscATsOJv+sB5CG/Se9ou3vLu7P53DE5rW9rCoSsEUIAUWSliRvrxkJlNkwtlWfUqTQadWRCR0rD6kRCfuyAjv1+M/aR1JlYCMlDdDbIpU4ywzAMwzB7x7KwABbjWwQHjt//XwIJH64P8RlDeQ0EWgkggaJgAPStgWwBLGEBtOedXE0ECquJxMvpIarahAiERMhpuDGfy3+gaX3vn13O2BrIMAzDMMuTZWEBLCYOYGzFa/s9W7/Xlx07cjivrkwgjLUEQhIAaTfjFXi28BywYWPIDb8MxvKkR7Ja1wT4z2FN4jdjH93/ysffsqaJxR/DMAzDLF+WpQXQx7cGZo7rOqQ2Ia5MCHyVBoIJVWq2MCBbAMtaAKeuIeKkEwQCdOHGaIEoGxolRKSf2JWhC9Z8cusdbAlkGIZhmOXHsrQA+vjjA5P39j62+qfps3IRnak1PboqFIGMxV9hfGAhSkw8UYQFywxgYVo1anO7KDYMCgxJ79qlx+Wfap4+9lDitfbQZV+GGIZhGGalsc803ucAqIK9DXD1XekN/Zsyzx+N1H8GAE+1hCIomONITcU/YRFYioKRtLDQchxckZBEEN9ZGt8oaeTeUI78RWjK0qj9Cd9HhmEYhllm7DMCECbFSzz3QzzjCciu+mnfx1Vu/PCJSP+/OomiXgpZsAbapeTYGjgdMveOUBMiaCAMCDAkiLYLPfJggsb+kBAqhwJrSQg5OXxg2Q8jYBiGYZiVxj4lAB3OLUwAcvU9Q5ua7ux747iOTo60/vWqUASBwELYGLtuiBE+K1kIxu7eQvZjq59AIlFDoMdRjz0a6JHfJDAaFAITBHHQbY2F4IEMwzAMwyxL9kkB6MApt7Bo++mOn33qjoF/HFH6nRKpf1VCBAXRQ6pgDYx/srJEINkVla27N14YJCxI4oknAz38YAJz2wIBkhAlgY5X20Nk6ccwDMMwy5t9WgDClFtYGxG4HkC33D7wpbGxscNGI7o2gUiN1i1MNmrgSggijVb4ARYWToktoDJ291JuQOiR3yZg/K+BIIWIIYEGiN3CAjmcDsMwDMPsC+zzAtDhu4XbfzGWabx9+ztUpI7Nk767JRRBYtItbF2hsM+6hQsWP+vuRSQQIYEaQz32p5DGHg1RjyKKBMVGwVj4AcYRZqaC2DAMwzAMs5xZMQLQgQDKRvgTzT8b/FX9bTtPHIuiNwuALc0JEWA8S5gU2GWF9yFroM0HTVo5MSAgjTSxUeqxR0LM90uBgXX3aqP2EArCD1zURIZhGIZh9gFWnAC0TLqFzT8abh/61sDg4OHjKvp0AiHbHMRzXBWRcwvTsnULWysmTQo/QkIBgLLg7h19NITslkDEY/sC392LLPwYhmEYZh9lpQrAGM8tLPZ/AHbW37rrPeNR7pisolsaA5S1cmq2MCzD2cI2JE5hYZNY+BHEIm8M9djjIY3/LRQ0gSjicX6F8C8iXjzZBs1mGIZhGGafZEULQIcvBFffOfZI7a1Dp03ko1crgr82xm5hQiJSU8a0KheBdvXjwioeVv4FBKSQslukHn8sQPWUEPGqyaJ4nB8LP4bZQ2SFD8MwTFXBAtDDuoVjt2fd7aPX7xrddWQur9YHACONIUpE0NoukWYDqFSVEMTCvI04ffHSwICF2b2CKLdd6LHHQsj3SRFHcgmM8IulIY/zY5j5QVX4MAzDVBXBUieg2nAuXgIQeA+MAIxesevlie8KTHykVuLZZt9EpCNAkkZIxWa26nCZxrN7NcRGvVj4gQRSI4Ly/QHkh4VAgWC2KyNgtQCBSHrp080wy57W1tbm2traNxRvLyw2BIOZTObbS5MyhmGY0rAALIMfNgZvy/0FIHdO9rT60wDoqroE/kNOxS7WCIAkTU0UWXQxhZMTPDC2SgqMJ3iAzgudz0jID8p4nyhY/IB0wd1rzZ0s/hhm74hDRgkhOoUQXzR9wnilSe9bKZUBgO8s14lkDMPsm7ALeAa81USw5idjP3li4/jzc3n9Xgm0szbEwNb/CgrrY9Aijg8kqzonV/FAEaeU8juknngywPxOIVAQkii4e00CY3cvsruXYeaJ+H3P5/NKa53TWkda67z9jv8NAIMs/hiGqTZYAM4Cu5qIEVviOX+GXM0tE5/Mj00coRR9KyEJayTGq4kUuvwLu5oITgWoRir8RVCY3UtqVOjs5gByfVKAngrrApNhXZCFH8PML/H7lEgk4pG1ZT5yKbwDDMMwlWABOAd8t3DdXbA52DDxZlLRSUrTAzUhBkGJ1UTm9frxenWEhckbUFiZN3b3IuV6A8r3SNQ5jAM5x7NBpsK68OxehmEYhmEmYQG4B3huYRHcon4ebMi9MIrUBSAok0hggM4t7ETgXrqF0YV1wYK7V1h3LwFQNCh0vkeCGhYCpFF8cZiaeBUP5Nm9DMMwDMOUgAXgHoLTVxOhcEP0FTGcP5zy8GUhQAcBSigsKUeAtEdu4UnXs43NHM/uFYWwLmpM6FxGQjQoBRipJwg0IMZzknkVD4ZhGIZhKsACcC+ZNlv4LugTG/L/rnV0rNb650GIMohXE4Ept/DsrYGxiNNuFQ8sBG2mPOj8dkm57VJAVHD3anNA7O4FN6+XhR/DMAzDMGVhAThP+G7hcAM8IH+sT9KRehMBbJJh7BY24k9ZaVbWLWwnmxSi+hXCusTiT2ukaJfQ+QGJaiKe3Wt9wzzOj2EYhmGYucECcB4pcguDvAn+Z9d2fQQo/QkhaEKEICE22MGUW5gmZ4xQweoX671Y3RXm7BLpCdTRoCQ1IgQabFgX4nF+DMMwDMPsASwAFwB/beFV98Eg/h+8DxS8gDTcjCFIDEAAYWTXD4mFYEHqUSGmX0H4gY5QqyFJ0bAUoCC2+hFad2+sDln4MQzDMAwzd1gALiC+EMQN8AdxI5yh8nA2EDwuaigwKg6IVGExDyAj6tDO+lDjQqthgTqPVvjFEZzteEB29zIMwzAMs+ewAFwErFs4ttYFP4YfQA6OhAguB6RhWYMyduvGfl8inUcj/EBnUYDdHq/iEYdxjid5sNWPYRiGYZi9ggXgIuGvJoI3wxj+ED6ME3A05elbJGgEFCo9LkhPiDisCwqIZ/dqsKt4IK/isYSgfVdkiY+w+/f1Z4MrJJ8MU47iekAUfUrVC/sS+1p+VjwsABeZ2Bp4dlxBAN4Mj+dH6JOQxT7KCUFq0hII2s7uFcDu3lmCFT57ci7pvR8EBXe+KvHRdr+b1e0agOX4zPwGrrhuoKJ8QlGjt5j53dNnXS5ve3O92V57Ps4xl/MslGCvpmst5LX9NEhvZSdd9O77n1L1AuxhmdvbNO/tfRMl3uvZhDDbk/I578/Oedzm+pnDJUp1BpZdnR8sdQJWErH1D0DjDaCGXwHtdSF+EFFcAFoIpYlAAGoSsZcXkUjHIwKZWTIfy+4Jr+JWblt3d3cqiqKUlLKLiJoQsQYRc1rrUa31ICL2j42N9e7atWun9zuHLLGt2pAlhKyjpqWlpa6hoSHutIyMjNCuXbvGAWDc/sbHVYoLnd+5PmvhNdKT21paWpqHhoYmAGBiL64X79NajxHFhnqw3+j9u/jae8OCrDFepddfyrw6weaEHXR2dnYEQXAoABxKRAchYoqIGhAxNGWIiHYRUQYRNwHAXxDx8Z6enm1F74MsEocLwd6cW3oCdnLb6tWrG2tra8OxsbFoaGjoqQW69ryBC5cOJ1jLPUN/f9XDAnARsD0LdJNC1BnwNgzwgyjEfvl8bBOMPb12mTdzEC7DzsSS0tHRcZAQomaq3S18m329vb1/B4BchZ9Pq/Ta29ufnkgkTgIA83keABwQhmE9FulxIUS80Iv5NDU17WxqatoMAL8nonvz+fy9AwMDf/cq/2oUgtKzWJh72BkEwfMR8WgiOhQRDzC3AwDq7bEmn9TY2DhuRC8AmPw+SkS/iaLo1/39/X1F+V2Qhq6zs/NARKwt8azHent7N3uH+lYbSKVSL0LEFxPRMYj4DETsrK+vf18mk/m6JxKnsXr16paamppuRNSuPBVjrh9F0QFoC0jxNwDUdnV1HeKXyeLfQxwDirC3t3djJUHa3t7+NClloty5/HOa/UIInU6n/zpPzwHb2tqeEQSB9K9v/xa5XG5o586dPfNwHfOMjbBaa+472HtTMkFT906Mj4+nZxAns0H4Zaazs3NdEAT/BACnE9FRiLiquB4oha0XRru7ux8DgHsB4Oaenp77ACBfdJ15fz/a2tqSQRCsnqnMFr0zwhO7IplMvkgIcTIAvAAADgKAFgBINDY2bhwaGjoaAKISpxWdnZ2HCCEUVHhmRWkQIyMj/SMjI9vnK//0CqiBWjgI8kA5PYuGVAAlQkDIwhjeApsrHmnrtPb29sb6+vrnEtHTENG8j9uI6A+bN2/utc+0ZH1SbbAAXGCc1S9+H86E40jAVSjFi8xrFuUoihtKFCKODigAqSr6T8uPIAj+TwhxqNZaCSFkXLMIEb/8ra2tz9m5c+djJX42WemZFzoMw7MQ8fVEdKwRk2ArcvtN1pLjPyG07bw5TysitgLA4QDwJtOIp1KpXyHi94aHh39grYNQJRXDZL5NQyuEOFMI8S9E9GIhxGq/gaMSBRLj1Qax2+b1leaYIAgGu7u779NaX5/NZm/auXPnrqJrzV/ihbhRSvkPWmsthBBEpKSUUmttGtjjXDLds0omk68TQlyIiEe5vGmtXT5qveN3u0eJROKsIAi+prWOhBDl6ktKJBKlGhonkNZJKf9ULj9kb7Ipr52dncf09fX92k+/n54gCD4QBMEblVKV0gO2/McWyI6Ojhf39/f/ai+eRZyWtra2w2pqan7tWcfQXiu+/0IIIzQPLSMOZkucRinlxVLKK0w+EbHsEANnZTV/1tTUHAEAT5W4d3O6NhTKjHkXzgeAM4QQDTAl6rStBxzlXKQCEc3vTJk7ioje3d3d/Rci+i4AfCOdTm8tvuY84MrseiHEuZXKrPfO3Gs7uiYNNalU6lwAOE8I8Vz3rrhOrrVmrynzLOJtUsr1UsqzlFL5SuXTT0NLS8vjIyMjR1mvwh63gK6tpRC+ggLeTBKiUM6ocQpiX0AAIZxpOrWXA4j1uz+T+N62t7d3NTY2vh8A3iqlnCwX5t6YOuWggw7akMvlPrJt27bf7EU5XDRYAC4Qk+5eUyD/CfYnAVcAwFvigpKjSJsChSKI4/4VlnBjk99egEWAZ4HxGmf3Qk5ahpqbm1ubm5vPJaK3CyEOBPtCm8rT/sYf2yH867mG236T+4ZCY16LiCcCwIlNTU2XNzU1fX1oaOgLXk93KSoHlw/d2traXFdXd76t7NfBVEWvaEr1lR2f4+fVHGOFo2kszWdrd3f3N3O53DUDAwO99ph5a+hKPOtpz91ZNhsbG9taWlr+Wwjxz0WNN9p8BkZEuiyVulfCxl4vOv9uSSpOX6nv2eQrCILislrMd4noTU4UVTi12REVThmYhu1Xs0lDhXMZkXuaadTNu2HO6+0PrOA8uKur6/De3t69afzc83ilte5K28EqhzlIaK1/09/f/we3bY7XnBwi0NXV9Xwp5QdNWXaNuq0L0KsLpPuh/wCoqLdk/63ttylIBwshrtBaX9jd3f3VKIo+1dfX1++Vn72tD+ZUZu2+eChHR0fHP4Zh+BUhxGHF74p9X8zfprOVr5BOPT4+/u76+vpTrPidqeybcmM67Ickk8n1mUzmkj2tJ8h62OhMeBEE8BaI4nOElVpVLKy8oDEBAWXhO+Im2OAZbIoP1fvvv/8piUTi20KI9iiKbs3lct9BRNOxywKAaTvOlFKeV1NTc8batWvfs3nz5k9XSYe/LDwJZJ5xy8GZQvSnZ0NCvQouBgkPYQBvIVOXKFKEEBS6U5PCj8XfXkIVyOVyrsIiz/UC3d3db29ubn4EET+OiAdqrZX52Ap7cnJDKWD3HdNmAsbmwsL5TGOZEkJ8cNWqVY90d3efV5SWxWLS5dTd3f2Gurq6R4QQVyPiuqJ8TxvYbPM1bcB2cV5tw+DyqxBxfyHEhxKJxO9TqdQlcUVcqATlfGSk0rO2aVRdXV3Pbmlpud+IP/MMTENjn9tkvszHWYlLvINubF+ly5W6PpT6ni1RFPll1SduRDKZzD1EtNl2QFSFdJBtyM3fp+7luKT4d4h4qju9fy3bFzLl3IjAU8vcz9kQ/2bNmjUHI+JzrQiZ6f4p+zp+355jrmVMuA5RKpX6vJTyl0b8FYpzLPzIim0BZTqak4kvvS9+T+zzcudcJYS4JAiCh5PJ5Fs9V/De1gdzKbPalpG+ZDL5ujAM70HEw5RS5l3R/rvi3nn7KddexekfHBzcorX+oLXMR5US4Drj5p5IKS9KJpNH2rK2R2UnXi0f4fNQiJOrJ5dcLfMfESjzZCkP/ajhQirdaYm3HXDAAS8Kw3ADEdVOTEycsnHjxlO3bNnync2bNz+yefPmxzZv3nzL5s2b3z4xMXEwEf22pqbmU2vXrr1oL/KzKLAFcB6hy0Hg+kLBi86CU4XAj6KAwyBPQDkwNbsEKIz1i4Vf1RaL5ccsLYCmvEerV68+oL6+/qtCiFM8a19cwZVyeVIcrLti77zcbDbhLIW2oeoWQlzb3d39ytHR0bc/9dRTmxephxhbxNasWZOqra39shDilbPIt/Y0jJ83V3mXyrdwkx5sA9MhpfxEd3f3q6MouqCvr+/B+chvKQugM6DZ8TnPk1LegYgd1hUVwpS70LktZ7wMzN6aslv6Sn3P5nczWADNvcsS0QYhxAXmPlpxshs2r9J+P6ezs/PZfX19f9oDy1x8fGdn5zoiOsKlo9jy5ZUhIwDX7+EzjicQhWF4qhAiMM/O3JJS92+yYCIGWutcFEU32l1zuW4sijs7O08IguC/hBDPtOJJ2U6OnLrM3CvrEtbB+BGT7SwBQCoIgq+lUqnTlVLnWWvg3rwfcymzgR0jejoAvNG+t8q6bcm6NctaN8sQi51MJvPFVCr1eiHEkf6QnOK02HMKV1YR8Vo75hDmUkYnXb9nwPmYgCNNWzvZESjdyrqV+AkEBJiDi/Bm2F7G+heP95NSftukcWJi4qU9PT0Prlu37hIiarB5M9cYyOfzd/f09Py+paXllNbW1l8HQfCp7u7uu8y2anUHswVwHnBr/xrxlz0TDtZn4/UyED9BhMN0jiLSsaUvmDycpd+iYi2AsUusvb39RfX19Q8Y8ef1dgPr5vB/pm0DaypJaSpGKWVgvt3H+7dzU7nZw+SNn5lsqJwFQAjx8oaGhgeSyeRL59MyVgZpLWLH19XVPWjEn7WIlcr3bnn282fZbbv9rZqe3YIYsxbQI4Mg+EUymbxwIcQuTc22HW5ra0smEonbjPiz9zr0LA6TYzmt9Syq4AKuNgvg5DYi+r5zj/rX9fGEbtwASylPs7vmWufHx5v3RUpZ41lSp13LpMcm47D29vanuV1zvJa21zrDCYZyAsZZQO1z/1V/f/+Te9DIUiqVujgMw58h4jPdsA97X6FYBO0NLi9+fWDya+ogKeUrgyB4oKur6/mLUB9Mpsfmb43ro3rlaW/zbfLlOij+OM3d0oDx6lexC9/ch6O6urreZe/BrMqps9rRK6CdAvgI5d1o+jJljyaHACkMIaAc/ARvhuvKiL84DfX19W8Iw3BdFEWfNeKvq6trrRDiE0KIDxHRWxDxXET8XG1t7SNr165909DQ0KBS6iJTWYZh+KE9uoOLBFsA9wIXN8gUnP7joXFNF74HUVyMAhp1Nm45Ci96XJtY4Vd1fYB9G1P5BEEQmj+TyeTLpJQ3AkCdN4ie/ArKCiPpLCta62Gt9Z8R8XE76/UpIsoRkTlmNSLuDwAHA8CzELHNWUKs1dC3AJJznVpR1CWlvDWVSr01nU7/zwJZAmOLSjKZ/Fcp5TcAIPTGb7k0ufE9seized6ltX4IER8CgMe11v1CCLPN5K0JAEw+D0TE5wDAP9i/Ay/f6E2ACOy2IAiCz6ZSqWel0+m32/TNS6/Ya8yaE4nEd4UQXe75euKQbIMU2J/USCkhiqI6Ly3TTgsFMeLMKSUtUeXSU+q70k/scTCDBTD+dyaTeaC7u/tviPgMZ2GpcF+c6DgdAD6xB2XMHX+669iUyo8TZFLKhNb6FAC4Zg9CAtGaNWtSiPgCNy+mlOUIinoanvt3Ttfr7u6+RgjxdtsJUM46V+6a84EnzMm5Ve07eWAQBD9PJpNnZTKZ2xfaM+C9+5P3cZ7yHVv1+vr6Hkwmk9cGQXC+eRet1ihlBXT3xNwHU34+3NnZeWNfX9+mWdYP8dg/HcLVKKGV8hDFxhYqY2gptME6DrKrYBg1nO+luxjXIXlDFEVqYmLiK9ZlPWE7j5/evHnzf5q87b///h1hGN6NiB8AgP/ZsmXLT9atW2fe0dO7urrae3t7B/bini4YLAD3DKTLAa27F9S/iFejFOtRwMGUJdCRm92LCJotfkuJqVyiKNrZ0dHxXCnlj4z4c7Pjiio/97JLOx7uFq31d7PZ7L2zCW3R3NzcWl9f/wIhxKsQ8ZVCiDYriCZnMdJ0a2AslKSU30qlUvXpdPqaeQ4V4yx/50spv2yvPdnI2UqXbPriekApdbepvHK53O3bt2/PzPI6tZ2dnc+TUv4TIr5aCHFQUb6dZcFZO85LpVJt6XT61fOYV2EbkmPswH3tT1LwRL0T55uI6HHzDQD32cN2c/3Y3xqxP2ryM4NlxjTYdVgcm2aqbI2V+6FnwUOllPKvXwKThoiIfiSEeK+dZCDKiSTPNfv81atXHzA4OLilQh5KJq+lpcV0dF5sbqu5VqmG3Ps3WbfiNXMUMLHgCcPwpUKIejf7t1y+nMVKaz02MTFxk901m/KEdhzsdUKI13hu5rJicyEotgba+qBBCHFTMpk8c6FFYKnOyTzmPe5Qjo2NXdbQ0GDqwi5rrd3NFezdh3jMoJSyMQiCL9kOR0UBODnx4ww4hiS8haJCCBvb2u6eF5o8l4YgnvhxqbgZtpSx/sWYsk9EhwPA3/r6+jaabWNjY7qxsTHwwjVFW7duTa9bt24LIh42lTz4aRAE76ipqXkuANxVjRNCWADOETobJN4ACtcD5f4lPDwI4KMo4BWgCKIsRViwDMU9kHgsKlv9lgRX0ZjeWk1NzTohxDcAoN4Xf15vfFIEaa2/R0QfT6fTjxSdslLjr3ft2rVz165dtwLArR0dHR8Iw/AtAHCREKLdjvcptgZK6xI2vd6vJJPJXZlM5jvzVElIa/l7axAEX7bXR78CdrP67KzOX2qtr8hkMnd658AiN4w/7s/fNmHH9j3Y2dn50SAIXg8Alxgh6Nyr1oImrMDJSynPSqVS16fT6bNhHmKh+VYVX5y4Rsd8lFJPENF1RLSht7f3UTtzD0rkzxGnvbe394Y1a9b81MWbK8bF2kPEdWEY3l/i/ph7/WQ2mz2+3Dl8tm/fvr1Meqali4huIKL3Okt1KfHiLKPWMldbW1t7CgB8bQ5lLD6uvr7+eCHEKjc2roIoEwXtjS9qampaMzw8vGMW1wA/v0KIMytZGmHqeceWT6XUPbaDNhtrUZyfVCr1nSLxN2vrVyl3ezlmOl+J+sDc31BKeWN7e/tLBwYG7q9G0TBLcGho6KmGhoaLEfG7bkJIOVewfXbS1s+npVKpV6fT6e/PkP9CCA2EL6AA1FGsCEVJ69+UBy7CAALIwn3yJvhSBfEXp6mxsbHNWrU3uh0NDQ3mueUQ8aS1a9dmhRB508EKw/AluVzuWlcOEfFJ+4i7vXNWFSwAZ4m/ige9EtZQc3gpEb4TBYQqp1WheBWEnxvnx2u4LR1eD9dUql9DxGcY0eOLP2cVs9ueiKLoP3p7e291p/AaFZqldSEWTDYg8tVr1qz5n9ra2qsR8Q0utIKzNHjWGaOTtJTyW11dXdt6e3vv2UvXqLDx/c6QUn7NE3/Cy7frjeeUUpel0+lPFf1+Lnl2v8G+vr5RALh29erV362rq7tCCHGhn2/7SAIrAl/V3d39zZ6enjfubSPnN97O7WmtnSaPg0opI26/akNeOLBo5ZdyjO/YsWN8pjSsXr06DIKg5EogAJDfsWNHek/zV4RzAz+USqUeFUI818ZDLOsGdr9DxDOsAJxT2bIhUWiW1zENeEtDQ8MJw8PDP5zDs6XVq1e3IOLx1m1d0tIIRRZT02Fzf8+QrzgdyWTyGinla+ci/opEHzkLlzdD3j9Ue8JflLg/0yhyCUvr0q9LJBL/19HR8YI9HNtYDcQdzHQ6/b1UKvUmKeXLZuMKts/e1Befa2lpuaNcYG/XHqsz4G2iBp5POYjQdHwriz8df2vI5gnePnWq8rhg1n4ZMK+BeacR8SgAeI6UslUptTOfz38wm81+Zpb1SlXAk0BmwA/rYv6t3ljzVr0q8TCG4t1aG/FHisBWBBzWpWrwBEEDIh7si69i8aeU+tGuXbteYMWfgOkrSMzlJdbetH9pGn0jcLTWbwGACXt9VTwOyKYlkFJ+v729vQumYhXOOds2ltkhQRB8x7m1/Xzb8UamEd9KRCd54s9f+mquFZf2VwAZHBwcSqfTFymlXgUAQy7f3lj4wM7OfUMymbxiLoO+S2Z66lmDbUCUzeMfiOgfM5nMF6z4K17feabZ3ZOXqPCJy0sQBDXlxv55YXMqnWcu9Ya06f6BvYYuZ5nyLXMAcFxzc3PrHJ5vHBwYEU+2DaGsZAHz3iuSUp4xy2uAeyZhGB6PiGvcO1LJ/WvSopQaGhkZucVLa6Xzm/fioiAI3q61nlH8eddxIt4NERFuApQVeVk7RCDnxtF6E6TQSxeVm7AD08fgOStYWxAENwBAYjHbE9odba13kR0GEc2h/MTH5XK5dxLRuLsfFcaQOiug6Wh0NTQ0fALKh8chOh3ahIQr45h/lSZ+4KT7V2MIEiK4KrEB/mTb9XJ5cdt3mPICAGvdjvHxcVM+G7TWX9q0adOaKIruQsSnNm7ceGUmkxlz6dVar7XvTabonFUDC8AKxGFdYgsz6PHX1x6r3lxzrwjxa4Jw//wERbbbIp3Bl8f6VQ9UNOvTjcErFn9a68+m0+mz7Eod89Vzc+ICbS/4m0qplxHRkHP9FllmXKXfGYbht+z2PS1LtUKI7yFikxv75nrc1roQENGfoih6cTqdvt9zbc+Xm2lSCGYymRu11icRUa9zccGUCgzszL/LOzs7T5uPeFlO/NnQDH996qmnTk6n03/x8li8xumsTz3Lz0KfwxHnYWJi4oc27mJQTlx4QkpJKVfV19efYHfNVPfH+9vb249CxANcmZ2FpcyFxXiJKYtzud9BEJzhzUgtazEzRVmIOHk/tYHVK+UFbaDjY4Ig+JSd6StnEn/OBe1uqxN0WutHoij6tNb6HK31EVEUPWtiYuKZ5tv8O5/Pn6WUukop9aD/O5qa1VzWjWz3k+sgSSmPSCaTH9vbDtJs8LwS7n64aABO8CaklAEAtM3hPY3F2/bt2/8WRdHV9r3UM5RVvz48t6ur6/ji/DvhRkb8hdBOOrbslXf9UqEMxK7fHDwKabi60rg/n3Q6vQMA/iiEOHi//fbrNsm0LuDJ3+ZyufOFEAetW7fua16+TX5OjKIoGh8f/6O7p7O8b4sGC8ASGOFnvnE96LHX1e0XvbXuvxMh3iukeHE+Ryofm5IxcCshsfCrPnB6sObJIK627oms5e9jPT097/asL/P9gjorouzt7b2XiIzQGbViVOPU7FWy49RMpf+yZDL5b3tQ6ccus2Qy+VG7TFreC+uAXjyuv+bz+ZfY9T8Xcn1iZUXgQ1EUvZSIdjiXN0x31VIQBF9vbGxst7/bo3eJpmb5mvPms9nsa0dGRgaqdA3mvSVuYHbu3PlnIvqtFSsVRZMTmFLKM+dyoTAMT7evzYz30AlEW7b36+rqOtrumqkcO0vjKbZMiFICoUg4+O7fmaizHSthNU7F2cWu6vDHPCqlfqiUenFPT8/hmUzmPT09PTek0+mH+/r6Nu3YsSPd19e30fy7t7f3R+l0+rJ0On2M1vpIpdR/m/Jorxm521TBYgv2eboO0oVGvC5kQOEiwaus8HPB7P+stf6u1vqqKIo+SESXz/F9iuux3t7ej2utH3MisNw98F3BtrxeY8rG5H478SN/GhyNEs6lfKGesTV4qVm/UKgXYgewBgXn4e8m12OeCVdurwuCwIjgN5skhGEY2ugA8RA608nM5/NXJhKJt65du/YV5v6sXbv2xCAIDiWiO+xqSFWpEVgAeky6e9eD/u2REOb/re6imnp4SIb4b4qA8nnrWiJ29y43XM/bDogPlVJfSqfT74cFXJTdI66k0un0/VEUnW0XuNdFItA1fFoIcVVjY2PbHNIUi9e2trYjpZTvcm5er0FW9vupiYmJM+0YxbmG6NjjfPf19f1RKXWWs8B5ItCNeepsbm7+zF64vidnctsVCL6/ffv23+2j4s/hrJrX+5MJSh3o3MDWQnqyEUSz6Oy4cW6vcKJpNpMk7PWUFaSn202Vfhe3Qclk8mjf0ljh3GTF0UAul7vDS2u5c5tO0fuFEAd7wx8qiT+0HabIdqD+ppR6eTqd/v8ymYybMS79VTKKhgO4ffE4zXQ6fZ5S6oVE9IBbRq/IEzENnD57PP53GIafXWjx5w+dIKIhpdTV+Xz+sJ6enuf29PS81oraK9Pp9Jf2sKOci6Lo311/hCpMCCkaD3lIMpm8tLhDLAP4JAjP6kcl6krP+gchBKDgi3gz/Gq21j+Xz3w+/40oivqDIHjffvvt9/Rt27ZltNZ/B4A+N9wnl8tdHUXRg4h4bktLyyop5efsbz/ssrYH92zBYQFouf5skM7dG51b/7LDj2p4IEjIzyBhez4bj39A6+61XbTqfKCLRaWxQNWGs4JZ8RcopW5Np9PvXOQZdspaAm/VWl9oXbGapk8YcONf2pqamt5fYfxLSRKJxKddEFtn9bTndcFW37Jjx46/LFG+71FKvaco3+C5e/41mUweu6fuLpoe8uRat3nec1M9FMYkK3Wj1jrrhE0lN7B1j3d3dnYeY3eVu89x3dba2vosADiUvBUbZkqUs7jbYv2KOXQ0nPu37Pg/bwyZScetO3fu3DVDHnQymTxACHGxDQ3kVqopl3ZXV0SmntBa3z48PHyMF5LFD3quvOEi5Fn7lZdftNav3/b09BynlPq6daOqWYpAaeNZHtPV1fWqudYHswF3nzT1i1wud1Q6nb7Urq2si5d93IPLaBsb8C6t9f9aK6DyYlTuliZbNwgrAv+zq6vr2eY8V1hxTTqO/VgoI5UmfkC83JukHGyEcbjMWRDncou2bdu2M4qitwghGsMwvGvt2rX/sHHjxqdv2rTp4264TyaTGXvyySePmZiYeHdra+uPgyD4hyiKLu/p6XlwgbxL88KKF4DO3XvODaCy5yWekX9H43dlQtwmBB6Rm9BRXiNNunvLBZdkqhbP1WMqEqGU6hkeHn69273IyXGWwC8ppX5oGgO3nivYBshWekYE/pudEDJTxRFbOVKp1MlCiOPtmDDpDeJ2E12+mclkbrSV+GJXRnEYnEwm83mt9V0u3wDTgjibz5X2+Dk/F2+Sz6Z0Ov0bu7kqK915guzM641E9Eu3BF8lN7DdPxs3cFwnJhKJl7vGGirMZJ2WqClro3nvDuns7Hy2S0KZn2gbIulU3z1b7txOsCilvj9DUtwQkEuEEPXFk8DKpNvvJN6STqdPt2ODpTfBay6QJ6Dy6XT6XKXUF20nSJWzgsH0sZtx2owImu/ZwM5o7Fn+7u/p6TllYGDgCbcOuj1UF4nePbocFMJAvUdrvcN2UCu5gsl7XgkhxDVQWGNQXw4g5Ab4CuXgVgzjWcXKSvCp89iJH7HrFwG1hgvwzsLwmz1It9iyZctP8vn86xAxGQTBw2vXrr127dq1J6VSqf0PPPDAzrVr1x5z4IEHfqSuru6PUsrjstnsRzZt2vRhz8NUlaxYAWjdvXEw5/TpUJ9/R+OHpKz5XRDiv2Qj0NmI4nF+cRA/J/xWuPhDmgyOOy28xNKmqjJFFYwRV2+z8cmW6sWMK9Aoii4gop0lZiY7V2aTlPKtLt0VzufE43/6A+jd+YzoNRXu8PDwe5e4JxqnM4qid2mtc248YJG75/hkMvniuVo6PCun+Z0Rf7kVUre52YbXzzTBwGtQzXEvn6EjEG+XUp7ujw+bTYK8dLgl6F7up7X4cPO/jo6OQ41YdOKxgkBzZaWnt7f3bj+tpfLQ2NjYhoiv11q78jST69dNDHssn8+/2s54xXkYRqC9CWH/YZ6X7QTpclYwL13S1gdHdXV1zfndmE3abN6HxsbGXmODG4s5zJCfLa7D0q+1fr8dqqHLCfIir0hkRFUymXybSe/dbulVhPNIwZCdAKInXcHecm8QQkB5+HawAW6dg+u3mFjEb968+bp8Pn8YEf1ASnleGIY/q6ur24KIvWEY/koI8QFEfCSfz5+0efPmD1Wz5c+xEirJ3aApdy9lz2/5/zrXNf82qJHrNWFTLkuRDUMq2N07HY2Fgbkifs1w0Gy7YqkTNQOeFcxUJD/q7e29ZYmDq8YVeH9/f5/W+mo7mFh7VsBJ8SalfCMAhJUCoUJhof5DAeAE09AVrQ+rrUXxk3a25LxaEOYIWTfQH7XW17kGAKeHxDGC9Z1zPbE3ycF8/95tnvccVB+xMMnn8zdprcdcmJYKbkV3zw9ua2t7nttV4rzU1taWBIBjrIVoVu5f/1qeGHXjAEuV4bj9kVKeZstDVMkiZt2U5rQ32ZVVyrVfsZuwqanpNUKIFm8MbNlz2/ti0pzN5XKvHRgYGJnnTqJzE2MURW9RSj1h86wquYI9IWTejfPmKS2T57d1o6kjvjY4OLh1gT0E2noCvqqUus/zgJTsuOAUwqbzY6Zc3gMQXQ8g8cewlSK4GMM4BIyeHPFXiPWnUYCgPPSjhous63dvnmWc9q1bt/7pySefPDuKopRS6gyl1IVE9J4oil6dzWaf8eSTT7548+bNP692y59jRQlAI/zMN94AavTf65+X+4+WmxMJvEEgHpLN6sj0RwAwcKtAsfCbDhKaNxV1oan9u9n2nCpuaGkqjpep4HLZbPYyt2uJkxZbBPL5/LVKqZ6iFTrIWwP1Ge3t7S+wvyn1rroG9DW+qw5gWmDZ7WNjY//lXXcpie+7Uuqz1lUtiqyz5vsVs3R9T53UWk6hYA17coHSXq2gDTL9cyGEaTQruYGduw8TicRpdnNxuXIx+V5iXadlx+SVwyvD5ppHt7a2dpd55+LyKoRwlsYZrX92gsb1MyTBuazf4MpGuTx4QyW07SR+YmBg4JEFEkLOCjZKROf7EyLK/sC7lwBwmp0cNi/p8i2MURR9e5EsVs4TcIGNVFA2NiB4+TedWyHE6jAM48kV55hdBVfw1ynruYKn4vFqkCAogovwZpivzu+kJXfLli2ZjRs33rxp06bPb9y48dObNm26ftu2bU/Y45bN6i0rQgCSMxnfAGrorc2tuQtXfTIRJH4dBuK0bJ7UhCJdEH728EJhrFphsxTEFtNCXSrGFIHO5R4228+u4oLuzQw1z/LHO3bseLyKemY4MDAwQkRfRy+YL05fxoyCIHDjtUqVRxdr8ExXUXoNXRwrzTSWNpp+NbzrcQHq7+//AxE94AfG9sZpNoZh+Ap7fKXl96ZOOjV+y4iJPu9aK4G4XNgxcZPW1JIHTllTwLPMlXNvng4zLMlWNkG2/rRWm7pEInGy3SWL093Z2bmOiI6yVqBylrDJMZ5G4Pf19d1vd5VbwgtaW1sPQcQj3ISoSvfEE39bM5nMx+bJ7VsOZwW7U2t9h7cEXKW4h2iXjGxpamp6qd211++zf08HBgYeXYRoCOB5QP6gtf6cFxuwkhXQdwWf09nZeYa5jycUuYJR2NU+AJQRhJSDn8ib4bq9cP2WS78uMeu7eMzksqAaGoUFIx7nd/nUw89d1PLm+pbgoTAh3qMUJrJZHa/iYaq4SXcvC7+SFLrRpGsEYl7T3wce2GIqDNhLs/qC4g8az+fz/73U6Skivm+5XO46P2YfFDXUiHi8Pb64QYrL6Zo1a55px09B0TJzbnbfdxc7YzPgZiff4No8L71gRYcbNzarsuXfr4mJieG5/HYfIK7bRkdHb9FaDwFA2aDQnlA2+w5rb29/ut3l13m6vb29UQhxgluSbS7uX4c/h0EIUcoN7GJzvkxKmbDu3UqWRm0N5DcCxHHcyrVd8faampqTnYsVKkxg8caPmpN/ybqWF6UNIKJPw+7jlHfDiSB7L18+X9f3xs3+wU3Umq9zz0AsoDKZzHqt9SbnvSh3H4pjAwZB8EVTRu8BUJcDBM4VDEG8FJwCBEEKhrMRXOBOsQB5KJ71Pd9jJheFfVYATq7isR70+IWr/zF3cevdYSL4hgBYOz5RiMVUCOtinxm7e8uCbm1LAF0nBQLpHxwFkKcqLj9FbqMt/f39v7C7qqV3Fhe8HTt2/IWIHnbj2DzQ1v2HNDQ0dJT4fVxewzB8QQn3r7b53prJZKptRqwLX3KXDc0hvWclbL6PtktgzSnN5hYGQTDm/rkgqa9OxPDw8A4iutNonkrLbdmGNDKiKwzDl7nf+99BELwQETvLuX9p+io7BCWsjm5Ch9X2x69Zs6apqIF0S6ud6U1kKueiBRfeaBbuX7LnffFMFkw3RMQuPzaay+X+1z/HAhLnPZPJ/Fxr/aRzw1aaDOK5gY+eRxejGzrxN/vvxXxnzLVG8/n8uzwNXMlK608WWxuG4ZUm/et3dwWHGMQC8NK6W2DzDMu9rXiqtgHfU6at4vHvdancJW3XykDeHwbi+PEJUrlJd2+hdwqFLtBKaizmjKZC7CSJKEciNZ5XFMdZq+YXy3lObA/35wCQrcLy7tJzl+2Na2/gsxuY3tTc3Hxw0fFTJxDiSNvQkf9TW2f+2ua7msp3XGb6+/v/CgDbvPSi5wbav62tbZ09ftazT6EwIaJqy+RCY1fGQKwQGNorW+aeu+EFuugYt/rHbiKDpsetnBSHxSLLu44Rke1hGB5rd02uxWzHsx1bydLovQdCa/3nvr6+35ZKs38bbJl5jjOoz8L9a75/sX379swiTpQS1pJ5p//uV0in+6xbvXr1fm7XfCSEiHrcn/NxvlmirSv4Jq31jS40TjkroCtqngh8Z2dnpxHD6hx7Hyby8DbQEFEOfi1vgi/Ns+t3n6TaGsQ9ZtLdux709WeDzL+37T+ChoaHwwSeFymA8ay2C8Qjr+IxF+K4DEAaSDcHUkQKPn/QrzZupuoZS1cSv/3TWv/Sbq7K501ED9qGaloj6OK2IeIz7CY//c5q8ixnLfQafWeV+Z09ttrec5PeLBE97o878hp7GQRBKfdkWdx9C8OwKp/xAlMY4pLL3am1HvCHExRTZE16YUdHR6dnCdJ2ndyXuSXZSgk7z+o35opcBfFGRauCgCuPzc3NJ0kpm+3s35LuX5qazW7+/qFLY6WbYYQlEe1n81TWrey9K+Zzl5+2xUJrfb9Ny4xuYDuutzaRSDzNbd7Ly7tO19A8nW+uxPkdHx+/UGs97L3/lWIDuqEiwi4TJ2+I7T4g6m+FbToPb1ME7/LPz5Sn2hqGPcJ39468t/Wlr3x6xwNBrfw8EHaMTejIriAowdZq7O6dLTZCL5BqkCIYyqnH5fjoR/Ygmvqi47sUlVLVuhi3q6AedysVuHrPG/dj8rKu6HjwGu1JawBOTSBx338p8btqwFmNnsSpEC7+gG/z77X+sTOecI4TFfZBxM6dO3dprW+x96/kxAKcGuccT7iRUr7E7oonwbW1tT3PdDj8wMn+7/1xqkT0MSJ6qtxMVpqKa2kOP8Vew38Hz3SHzeD+Dcz7oZT6gd1VKSwS1NfXJxGxibzZ4eVwM4uVUq6ztFjvirvOY/b+zDjhyesQHmA37XWZt7d9bG/Ps4fEnZHBwcEtSqnLvQkhMy0TJ+yEkCNSqdS7TXlYb4+RN8M3ww3wAFS5h6paWNYC0Hf3Dl3S/rTspZ3fqU2Ed4QCjxqf0FFOubAuWCgM7O6dHZPBNBFBkwpQBNr09nX0L6nfZZaqspgT3qoauWw2m17q9JQhrqDGxsZ6AWCwSPSRZ8HotMcXl916AFjt2mSYbuFBrXW15tulNe08iDA9bA8KIdy4R67E5wAROTdwxZmv7nArwiZJJBKneq7badazoglGQ+Pj419AxMfdEIQyVkC3vvXTurq6DrebjYBrAICTnKWxgtvPWf8e6u/vf9RLd8ms2d+tcdakSuXHe1cirfWWGc4937hJYL1a6wmsEKcQijqE8/huxPdLSrmUbaK2y+V9QWv9iAuQXU7cesNchA0ldXlHR8dBzvXvFnhY/GwsT5alAPRX8aA3Qm32Ax2X1dXK3yVqxGuzEenRvI7H+TmDiNF/XChmiR9MEyhKSJQSKDc0kT9rv/s3/f7y5TOoFm3DNEZEu+y2qkz30NDQLiIaLHaHwtQMuBZ76LT0NzY2GgHY4Isof4yXEKJa8+3SM+g3ejgVPsT83ew2L0kKlx+xVay3t/cerfW2ooDg0/AnaAghTrQTNPJQeAanOitLcblxgsye4teDg4NDRPTzcsvQFY0DNMLlVLevo6PjhUKIZKU4g9Y96c57g908G0tZs7WUQaXy44mq0YmJiUH385nOP58opUYQccS5yssdV9QhnK93o1rqhdhirbU+1xPDZb01nnfH1HENQRB8xe5Ct8DD4iV9ebPsBOCkuxeAxj7Q9apoXfK3iZrgSqWhZSye3Qv+it88zm/2UMHlC1jwIVK+KRCBQNgxFunTnv7A5tuuB5Drq8+NWg7X9mV37dqVW+rEzID2wk8UN6ImI3VFx8fH1NbWBs7d61fmtgLNE9FY8b4qwYnbXClXj31utUuQruWOqc/HAeCmmdzAnjDrCILgOLPdhoU5wovJN+13nkvW7LsZCgF9f+JcmBXGbbnZ3ae6Zx8EwZnlJprAdGtwoLXO5/P5H9ldleofZ9GqdyFDZrhfLsljw8PDEzMcuyAMDQ1pT+TONAYwrg9swPT5oKraRq111nYwKoK7LxP3smQy+Tpv3WVmliyrm+UmeWx//5pn5a5I/biuRv4QEJ8zOqEjrafcvS72+1Knd9ng1k6MZTVFElGsCmWYVfSLXbnxF669b+NPCUCcs3DBURcSuUzKuWuA/EageIYl+d9aa2EtNeWo6neglMiAKdGbX5JE7QMopb5vRZmoNBvYWZ2klK+EgnA6RQhRM4P7N250c7nc7VCYzf1brfXf/bWdi6/lJjgg4mEdHR0H2s2vKDfRxEufsmXhgYGBgSfmMEN3rjN5xVK/K5UmgBTvt4Ht9yWwUPzkNxGx3o0/nfFHU7OjtRDi001NTWu8sdHMLFgODWNMPKV7Pehd61Nvbq6v/XWYEGeOZrWaiPRUWBeyFiye5DFLqGAuL7h7FSDBqlAGAcLAaF5d2HH3k8cfdF/6r8t0Or1rwBKNjY01S52YWRCW8oLZbycO0f92wWGL9jmrS2jHCEIVVohuLFNjmcHe5jPiH8vMCucG/qXW+gkXX67C7Fo3QSNeXUJK+TIoEzvPm6BjfvPojh07/mI7Vzkiut2WxYpuYCFEQgjxoo6OjoOEEE8rN9EEiiabAICL/TdTexX/Rik1OuUEqlh+XB1R19jYuCQW58bGxgQA1MzGXe1ui9Z61G7eF96NOKZhMpm8REp5lFIqqjSL3VFkBTQCsLOxsfFTLADnxrIQgE6AjFyR+mhTjfxGpKBpdEJHgCALq3g44cdWv1kRT/GILaZ25DapRilkQiCMKf3VsbHo8Pa7N32+MA24+mf8lsFZPurr6urcGLpqLR+mPmsoCgFD7ouI3OoW0yyBuVwuS0SR39B5lhrzd7Xm26Vnjb/RyzPZMZFQhWmvdkydbsrE/7kOQoXJIE58HdDZ2XkaABxeLiaffSYuruZtdrO0+zb4FscS15kst0KIU4IgOMu236qU2CyyNo6Pj4//2O6ayQMRX1tKudOFKaxUfjzraENNTY0ri4tV3twwjlZEbCxnDS/GpnfAP8cyBq34e5YQ4nK3PvhsZ/Q7I7XnCn5TR0fHS9gVPHuq/ia5pdx2XbHf1Q2NwaVG+CmywZwLq9Oy8JsbNGUhjSd5iFVSyryme5XOv7jtrs1v2/+BbT3X28p9uQ6o9ZYVC4goaTdXZTlpbW1tsrN5ixssVxe6Cn/asxgeHh5BxF1+8fdnTiLifouUhbni8rGfL15xarknw1IEp90XcB2B693ygKUO8ly8ZFdQ+SQRdbkxfmXcstIKlVvtpliQ9fb23qe17i838YSmVuQxp305Il7gVvco9U66MiyEML+9d3BwcOtc3l2lVI/WutJycZPXcXEnwzB0rulFFYBhGK6196HsesDgTdyxVtttbvMipXWhcJ6M/xJC1PrLWc5WBNL0ZS8pDMOv2PHDy/3eLApVLQCd23dofeodTY3yP0fHdJ7ICBMUk1Y/ZnYU3pBYMlPs7kVcFYpAEGwdyatzW+/acnzHz9P3m3tuRPUyHe83iVe5m0rhWW7zEierGFcBGoHa6uozgGkBd81ns3+8R5aIeu3ffsPrXKzPLvO7pSa2KCPi01woDiiaMBBF0SZ7LFfkcyO+X729vb/TWv/JNqTlxuaRt+b0IUKIcAaLnDl2m7e8oFs/dqR4NRv/954b2PzdhohrK1kMPQ3pu39nU4bjH/X3929zq8yUy7uXtriOEEI8fw7XmQ/cdY5wk2FmEoD2fql8Pr8vvBux6zeVSv27lPI46/qdk/iD3WMDKiHEM5PJ5AfsvalqfVMNVO0Ncq7Hkcs7D61JyM+NT2iliWRhhDgQW/1mTUFEuEBdCLpZCplAyI1F+jOju4YOb79729fBc7UvV6ufj2/EQMSj7OZqy1ecwEQi8WxvQfR4hy+GiOgJe7yffvfu/tUfC+i7vewycVCFAbChtbW1GQCe6cahwfQ8j4yOjj5pD622Z7YcEFb4/NDrCJWrL50FZVIoFQsm+1tly9lddkyqK3/u2d0EUytLlBObvisZSjX2Re7f4aGhoZvtrtmWYXO+HAA8VDRGdvcDC/l2M5RfajcvVsfXdYJOAm/5vlIHuntnj+nbsWOH6xAu13cjFuadnZ3rEPGqubp+dzvZFE4Evrejo+O53rKATBmqVgA6lEx8qSYQCVWY5Su8ZdyYCqAL5+ymRRNFtRJFYyBkVutbcvncC1b/rOfi7t8M77DLusEyHetXEteO2Mr92HlcQH0heBF46/l6bjRToU1MTEw8bo/z0x+/A1rrh2C6K4Rco0ZEz7cBd6upoYjLWk1NzT8gYpu/9JPXYP9ldHS0f6kTuoxxs8R/YFeYKemaxenBh9HfPu1kU0INoyj6SdG1lL3Wz7TWI5WuBVONtSh1HW+bc//+1JaDudT3rk37ZVHad79JXjw5RHyBDYMDiyAaYndla2trNyIeq3WsvcsKIHdP7PvxqA31s5zbQBeu5xohhFuxpaT1z++UlOugwO7PMpRSXutfiylNVQpAa4mioY/sf0pDnTh+ZEIrKIyTYPE3G0wnHCgOf01ASlh3LxH9ZSyvz1l1Z+a0jnsGHtkXhZ/DNTRWYDyro6PjULdriZPmo21ST/EH33szLs0xTwwODm4r81tT8f2yOA6bN+uyM5lMFi/Cv9S4+/8ybyIAepYqk6df2WOqJc3Ljbgc9PX1/ZGIHqpkBSwSZiVnC9tyJY3AGxkZudvumtYZ6evr6zdl0VqqKrozyyZ6yvoXi02l1Pfd+edwmjhdWutf+O9FpckpRBQJIRJhGP7bHlxvT4jPX1NT80YhRL17B8odXJT+++z3cn03nOv3TVLKl2ut41m/rvPrH1hOEJaylhbNCo6CIHhhKpU6nyeEVKZab0zh4SK8S4pCgGKtOajzjLgl3Iz0I9Dm71WBkKGAkdEoWr+1p/eoNXf13jC5kso+KPyKsUJIBEHwGrupWspQ/O4lk8kjhBCHupAYnhXPTeT4RZnxLK6R/z0RbfXHU+H0Rfj/dSkyVwFlxAQinuUvA0ZTq0+Yhv+2pU7kPoBbMeP62YyFK4c/qQgAHixjkXNlc4OzKu7JtcCbMKK13pnL5Vw5mEs95cZA/l5rvbnSkngwJQKltZS+raOjo3MRXIfU2traLIR4pyv3layUbsKMXbP4TrtrOdbdcTlsb2/vQsRP22EHAissg+d3CjzBXnat4KKO/1Wtra3d7AouT1UKQASg0UvbkgLxhLFs7MUU8cg/phyTq3hQ4c+oIUBRL4UYj+j6CaWObLlz4Irn/BlGnHV1XxjnNxOucrcNwBuq0B1q0ngeeksfeSJO2Ar/5go/N+/vBBHd4q38AN6gaHOef/YqwaXGid6ThBCHWMuHKBrI3ae1vsceXw1pXq7E9y6fz9+otc4hYlApJEw5PAFintEtdnNxu+GudavWOm9ntc5ZBPpjDYno1sHBwaE9bKPMb0yep70X5fLn3j8hxKogCD5ZJo/zRezJqq2t/YgQoktrrcqthQxTMTFji6rW+sm+vr6HXdIXKH0LiZv5/GUhRKsV3WVdv54nw+T9OqXUh83fZpsLLl58AW/ojHmeLbW1tV/wr81Mp+oEIF1eSJOur3lhQ62o17owY5WnfJShaBWPABFXJUSgNTw8nqNTm+7sf3X7z7b/1QvrsmIaVVeJmEpWSplKJpPnVsnssHgMUCqV2h8RX+sqQs8V6sRQT29vbymX2zSI6Nueu2vyGrbybKitrX2P3bbU+Y4RQrxvck7SlDtH2Wd148DAwEi1pHUZE9cLAwMDTxDRA0IUqtW51qSeRU4ppW63m4vLYlzoBgYG/k5ED3tj1vbkWs4K/L25/LYUuVzuO/57Uc4NbIldh0KI16dSqXOspTrY2zQUEYd76erqeoUQ4p3W/SlKzbyG6eOY3b28yU5wWY7vRuz67erqOltK+SrP9Vt2zKnLt9Z6OJvNXpLJZK7QWm8SQkyGzSnlCraeDxcb8FXJZPKf2RVcmuq7IevtrDIMDpcSC95KHUcvYQk4nd1W8WgJRRAibB/ORRc/eMfA0a13Ddzq3L3LPazLnuBV7q6Hf2lLS8tqWPpo8WjrOdOjbbDWP+EN9XGV23V2jeBy72ns2shkMve7htdVms4VYsXl29va2p65xK4Q1wCciogv8ScnwFSMOR1F0X8tUfr2RVy5+X6lGbrl8IYRmN8+1tfX92e3q8ThzuX8E2uFnpPL2XP1SaVUpq+v7y67a086rPFvBgYGfqm1/l2lMZAwfayZsPXE19vb2w8DgGgeRWAsWjo7Ow+VUl5nt2ElF7XzeHrvxjddkucpTYsJNTU1rZFSftGWDSwl4AAml4Ik+yyMkLtsx44d6ThwbRT9B06tilJygo9nsY7PL4T4go06sBzv24JSfQJwMpwFHKBdLCjBD24SmprdS4UaPWqQKBMCcVzR18dHR49ovnPwMycCRCvJ3VsObwC7qUw6GhoaPrnEVkAnhI6XUr7RhkDwB0Fr23vNaq2vcdmocL64MtRaf8qvGItmxdUmEolrvesvBfGqLEKIz7vH4onzeJym1vq2gYGBR9xYoSVK575EfA8nJiZu0lqPV5oQUQpPOJmf3DHDe+OGMPyEpgI/zzqhOD3UzIYZOj6zQdhy9pni96JMPp310exvTCQStySTySOsCNzbtYJj8dfa2nqIlPI2RFzlAlBXEDHkWfHNu3Fnf3//H2Du6xxXA8IKwM8KITr9zl8p65/Lt5Qy0Fr/MpPJfMmeQ/T19W1QSv1YiHhSoyrVqcHdYwPuV1NTc1WVeH+qiuq7GQRuaYAa0xQWtrED2OIsfrG7NyFRNIciyBHdr7Q+rv62nee2/mJi60p095aj2CUghHirdQmoJSj/sbBZvXp1i5Qy7s17vVWwHyNUjXD7376+vo2u8qxwTmcFvF4p9ScbT1AXVYKRlPLEVCr1QTcJY/GyHBOPe+ru7v6ylPLpTvT6vfi4u6/1FYucrn2duDHduXPnNiK6Rwgxpxm6XicClVK3zOJakMlkfk9Ef7OuzVlZAYs6LP7s373BvRc/0Fo/XvxeFB/szyK1Y1OTQoi7k8nka+y5nHiYbZ2B3numOjs7T6irq7tLCNFtzy9dDMZyJ/DeDfPPj3jnXU5MWv6FEK+vNOsXpnc6UGudi6LovMlYtlNrPV+otR6tNNSg2LUvpTy/o6PjH3lCyHSqTwBeMblMGS3bdcjmmzimC01bxaMpEQ/q3jaW02+rv/WpY5tuf+oX+8oqHvONE1mei+cbyWTyWUswLiQu2/X19d8WQhzohFCpMS8TExPrXfJned5Ia/1ebxjhZPBYT/x+uLOzc6HGN5UjsOOeLhZCvMlvAKyLxvTQpVLq2729vb+ZheBl5kZcvq2omrUb2HfJElEvIj5gd1XqVLp1iG+bzeoWjiLxtbGvr+++WVxrNsRBobXWlxa9F2Xj7XnpMNduklJel0qlvpNKpQ626dFeXmWZj7PSqfb29sbu7u4PB0HwU0TscuKvkiG2aAKEeXd/mE6n71+mlnFas2aNuY9f8bJWctavuyee6/djfX19f/TqBLJWwE1a6yudqJ8h+LiLBYthGF5j6yMWgJbqE4CT2PhUS52MpcUP6xKv4tEoUYYI+WxEn82PqsMb7hj6KsC+tYrHfONbw+z3KinlTe3t7V2LJAJdCCOdSqX+Wwhxul36yPWEXQWmbaV2xc6dO7fNQQzFeejt7b1FKfVDzz2CReObVBAE/5tKpU63rq2FtgSa80epVOodQRB8yog/kw6cWq3Eubd2RFH03mXq3qp24s7g6OjozVrrXbN1A1vBFLt/tdb39PX1jc72PYmiaEOlpd6K8SYBmT9/PI8THeL3IpPJ3KiUurOS27A4Pc4dbASGlPK1QoiHu7u7v97Z2XmCXWtW23v7/7N3JnB2VGXaf95z6t7bnU7CloXulnEDhSAjGkT8+DQooEKS7qDG7dNRlBEccQN1ZlQ2Z1AZxRlkFNBxQUXFiJAOmwI6QcEF4wyrgCIK9pbO1kmnu++tOuf9fnXq1O3q27fXdCfdue+f36U7fetWnTpVt+qpd6324sbGxr9pamo6L5/P/69S6oL04bOyXuco2x72MGitPX+O3grdHObz+cuUUs+sfODNLlghegNr7cOdnZ2XVingnx7TyzMej1ETQrJeEKXUCxsbGz8qCSFD7C0rgDBZ3MlMLrUXjKigKdBECC3fboz5RMOP+5MOEEPCb649Gc4oFRmm5YuBf7KMLwZH5PP5OxctWnTqli1bOtM4nRkYSvkCFos/rfXfZ8QfZWJeIh/zck9HR8e/T6FzCftODe8jopcrpRb7+BedEb/x+nJE9KPGxsZ3dHZ2fq9yjNO8z6a5ufnDSqkv+HIXmjK1vLz7JmetfV9PT0+XCMAZg/r6+rYsXLjwTq31GcYY68vCjErmZhqfm2n3j/FEiDuHNm/efG9TU1OXL3OStjwb87NpnbsoiqbD/ZvF7UcYhv+glLqfiAqZMIsRQoRG1pojL1zqlVLvyuVy72pubv4zM/+OmR9SSnXE+jr+/gJYRETPAbA8fvkEL6StzkbrdlG5fa/PjdY6F4bhx7q6uv4yyzsZVUP5dm8naa3fm836rXYeZcpXsY9pPmecB4HQGHOuUupnmc+OWCgz18pXg7hg0aJFP9yyZcsf5HojKnhWkbZvS7J73Untnmzm5SmwzH8YDKO35G/pOy0Wf/tzF4/pgCo6HGQuvPHPwF+Qji4UCj879NBDl2ViAqfrSTuNAbIHHHDAgU1NTTdVE38Vrp7tURS9PRvvMgnc8ps3b+42xpyZuZfYKkHugdb6u83Nzamb2U7TvqvMjaquubn5qoz4o4pg/FiE56Iouqajo+N6cf3OKO64Wmu/j3H6zgIjun8MlEqlNCN3Ig9Iyidw3OWt22O6gf2mrLfSPLp58+b7/FvTdV1zbsOenp4/RlH0kfh7RkSm8gGxkmxcbmrJjK8Z/nPP0lq/LpfLXaiUulop9W2t9fe01lcqpeIHnlcQUYNf3mbL0IyViZxxgcYPgzljzI1dXV1Xz0Hxl1IXBME1Fa7fqq7aTDhIYIy5qrOz8xdj7Lf1ruD/ttZ+O7UClhMjq6w/4/2p965gzFGr6rQiAnD24G1B5exenpcjrYl3hyX7Lzv6+l9cf/ugq41VK1089gQe6hs5yMw7aajYsvs1dQcR0fO11r/wtb+ywd5TvTikws+5tRobG//v/Pnz79Vat2aegqnC1QNf++ytE0z8GHW3M67gf8q4vLhCBKZxNhc2NzdvbGxsXJ7Zd2TimCazv0hjpJqamk5ubm7+pVLqnIzbV2Uu9OkN7u7Ozs5z5/ANbq7g5rZYLP7EWrtlPDdwxv0bnyf3bdu2rX2y3wdjTFv6mbHEZiaQP/79hhlKzkpDJK4yxnw3fvCIz8F0WOPMQ7YDVVrg2sbndfwwF/+s8jKpgE5FSWZdo+K/o+7ByFr7h927d79rjlqp3Pe5sbHxX5RSz5uE6zd+4Hi6WCz+8wTiHd3dMgzDj8UPzpnuH1VdwWk5Hd8m7uRDDz30HeIKrvGdnxVkyrqk2b11Aak6DRUZ/mExNMflbx24cMlG9GWye+faBWGvk7kIGGvtPzDzLv93m7kop0+OBymlrm9qavrW0qVLn50RQ6m4GUsQUiYgPBv8fWhzc/MXlFIbiegoL/6CSvGXxv0ZY87u6uq6PbUa7sGux5/VnZ2dlxljvhgLLX+z4wpLKPkxvUIp9aumpqavNjc3vzCds8xNh8YIdEe6v0hc3C9rbm7+gVLqTiI6dhTBG/ob3MO7du16nY9FlPN55lHbt2/vTRM00lqT1RbMJIDEx+u29PMT3I47d3fv3v1Ta+1OAMFoYjOzHVemqVQqrUvfmupOjjMu6ujoeLe19pf+exGmumyCMZEpyn+XR3vpiuXHs4CmBesj/9C2zRjT2tvbuyNdbNpnY+Yg7/o9Xmv94dT9PVqx66zr159v527btm3nBB440kLnXcaYT6ik0rkdzRVcURvQaq0/39DQsAT7vibsPkUE4D6CMgkeqfCLrxqFPAWG+f6SsatyNw+uXfCT0qPshZ9k906cjAu4IYqiu6y1b89cJLJuUZ3+TWv99iAI/jcWbt4tnIobO8ZFmDMB4dzY2HhkU1PTp/P5/P1KqQ97seliripurG6d3uXx0c7Ozq9MYxyis6J0dHR80BjzlVFEIPyYjHcJnwVgU3Nz8x1NTU3vW7JkyTE+2J1HC3QHMC8Wjc3NzR9qbm7+uVLqXqXU2oxbL8h2JrHWhn4svx8YGHjNrl27ts5RC8ecxXfYGDNBI9N71kZRNJV+vC7mkJl/oZQa1Q1Mwws039/T0/NAOoQp7dzEGIzFFTM/EJ+L8Tk5lntyJsl8D8viD8C2KIpO6+rq+v0cDYuIj2UuCIKv+DacGO1cq3T9Wmt/0NHR0TYJj0Bq2b3aGPPr9HpWrU1c9nrva8IuOuCAAy6vdQEoSSD7Bp+Z4NLTDaAon0dgDG+NIvvZoCO6ItiEkP2JSSL8poz/0i/u7Oxc39jY+IEgCL5orXtQHOGW8E+rC4koFm7vb2pq+gUz387Mv9JaP9He3r4lvoH4VcefqWtsbFxsrX1uEAQvY+ZTiOhEpVR+lOBvpEVOfUJEfEM+t6Oj40v+ojedxzm1eJzd3Nzcp7U+r3K/U7eID7q23nJxCoB4P9Dc3Pw0M/+JiDoAbPXuqfiziwA0M/Nziag5vbdba9O4VZV1fWWD2q21vyqVSmds27atS1y/exU3z93d3T9ramrqUEo1VUvQ8McsjVt7rKen58H0rUlsS3nL+wat9elpNEY191/6d2PMOr+NmUrGQnqz7+rq6lm6dOmpQRDcprV+sTEmjO+FFRbyGaXiuxF6t/RfSqVSa09Pz/1z9LuhfAjIBUqpF6bxzmO5fn2iWnx93BaG4QfStyexzTRp5P1KqV/ROLUB/a9pIuDbGhsbv93Z2fmTOTrfe4wIwL0Jl8tcEzg+2cjkAgosMTjCN8KB6OL6O/EUMtm9+3rI+wNRFBlfFPbKQw89dKfW+qs++zTyfSWzLuFUJAVKqZMAnORDMvubm5u3AtgFIL5hFIhoATMforWuS68tXvhFXuBlhVb6VBrG22bmHVEUvburq+tHM3jxcTe89vb28xsbG/+itf6CfypPx5d9Uk5j9GyaKAPgMKXUYZUrze4rEuEXZVzhKv17GtPkLYzxE/732tvbz8p0eZDze++SJmhsIKL3eDdwLruAvylbX57nJ5mYvMkcK7fswMDAj5VSJX++Dyt9UuH+DUul0o+yn51B3Heiu7t78wEHHHByQ0PDdbFI9XF7wyyjMyEEeXhVgtTz4B6MBgYG1mbKP82174YTXkuWLDmGiD6ecf2O2e4tLdJvjPnI5s2bu6dY/UB1dXXd19jYeFUQBO+LhafXNmMJT/Lb/jKAv41P12mZhTmGuID3AiPdvYgCTSrIuWzUe62NTlIbonfF4k+6eEwv8bc8n8+nWbVBV1fXtcaYU5j5T7EoyQg+ZJMgOIn0NplMvnlEFAuiZfHTrVLqSADNRFSXBoVXBH8Py/rzQoj8k/69xpgTvfjb05i/8WBfN+uLURS9ipkf9a4mJ868Cyq7vEr33++XGSPQ3Wb2V2XmHN6tbnxMVH8URR9sb29/qxcgc7Gg7X6DMeZ6f2qOSAbhTEcOa+143T9Gw62vt7f3SSLa5M+xES65NNHEWnvf1q1bH8t+doZxIrC3t3dHR0fHSmPMZd76rfz3NFuOZVrGkw3/8P+Ov3vKuz6/2d7efvIcFn9Ia50GQfBV7wHh0creVLp+jTF3dnR0fGMP9t15O/r7+z9pre1KQ33GCDsot4nTWj+3qanpItRom7ia2+G9DsObgEDg+GmaoAsUMLjdhuYcvd6emNuAjdLFY0ZJLwTO8tXV1XX3rl27XmKt/Zp/Si1b/io+p7KZfF4QpaLIZgQQVYogD2eEXyy6dlprP97e3v6Krq6uR2bA7TsaaazM3YODg8dbaz8PoOhjZlKXdLWndOVvjFUD3UeJnUmzjrW37twahuHxsQCtSByZMjwGYRjydGxjMsOJ/1cqlcYa1lTK+swEqRv4Hmvtk5lY1GFz6oXQllKpdG/2c5PEPQRZa2/21p4R28mIg+uzn9lLlGO/Ojo6/skY81pmfiT+nmYs11w55klvZGg/03+XrwfM/GQURa9vb28/c5oejNz4fKjHhDHG7Ol3JnX9nqe1fmnq+q02f5m5SOOw+8MwfO8ebh+poLfWfiQdzxjfRfiHEuUT1T68ePHiY2uxTZwIwJmDyVn9XAe3+PmXVQBNig2H9otU5BfpDbgGkC4eexFKXVo7d+7c1t7eflYYhq+y1v4sfRpPY0hGSf6gjKuTqlwssgkhSNcJoM8Yc00Yhi9qb2//jH9/b1vBnAjcunXrrvb29o9GUXQcM38rFoJ+v9NrQTnJIxW4Y1xIs/vrLujpuryVc01HR8fK7u7uhzOlcfYWe/VCXigUxltkttxY4uNcIqIbM+e6I3Njjg/+zyeYjTkabr1RFN2SJpVkj78XAoG1tjgwMLA++5m9SDn8obOz88ft7e3xQ+EnmXlz5juRTQQbVwyO9h1J69T568Ggtfbyvr6+5ZkQEEzD92NPz7GpfN6dQ4sWLTpCKfWpbNbvaFS0e7u4p6fnj9OQ8JJ2CLmOmX+aKfM16hj8g318buZyudzVs+g7utcQATjNULmIL5MP92NFsIF2UX8/IeAEtR4fpNvQI8Wc9xnpfFN3d/fPOjo6YhF4irX2u8y8Lb4wxRcQXzQ2bV02WjZsOZDeCz6dsSI8bIy5KIqiv+3o6Dhn8+bNf5rGi/2e7Lfq7u5+qL29/R3M/EJr7b8y8yOpgPMvldn3dLzleMYq+6uZeQszX2etPbW9vf3Ezs7O9MZO02npTMvpZMptqHRAuVxuWqyMkxkOkhtKtfGknU+CWWIBRDqOKIquT29+meLQ5GM/4zFv8MtP9abottPT0/OgDztQaRvAzLkTb/PuHTt2/GUfZ4On9eD629vbL42i6Bhr7T/H39/0Ic5fC4ZdB3xmvcm+KjPls98RAL3GmK8ZY5a3t7d/pLe3d/tMuHwz52GushxNBUF6IPZwk/l8Pv91IprnY4vVyMo5Q8Tvaa0DY8yvOzo6Lp/GOXDnjzHm/czc70XgmBPgl0EQBC9tamo6v9ZcwZIEMo0kfdvgm9kQx7/kFZvBMK+f2jF/A23c1gpv8Uu8wiL89jHpl912dHTcBeCuJUuWLFVKvVxr/SoAL/GZrgelN7CqK0meaIsAOmIhBeDeMAx/6rsapMJHZSxm+5rU1UEdHR2PAbgAwMVLly5drrVeAeClRHQkgEYAB6YX9GErsHaQiHqstU8Q0ab4Rg7gnvb29q2ZxdQ4JXSmNnhr1zLzPJ+vQL5frRtfd3f3E36xveoC7unpebqpqenF2fH4n4qZZ1OAuRtvd3f3fYsXLz42CAKdHS8ScUidnZ2/98vv6flqoyg6XSl1SHq8kKgDDsNQG2P+mh3XPiTdT93d3b0ZwGcBfG7p0qUvD4JgFYCTABzlY4EnJBD8dWGrtTa+DrQNDg62+aLayHw3pvN64K411tpLrLVfzs53NeJjEEWR6u/vfyL7+UniXOnGmI8AKMWCa7wPpMe+v7//T9O8/+4c6urqemTRokUvyuVyC8abg8x44gVL/k+z4Rq9V5h1Jk++CIougd356Wdev6BOvbGv30RgJPXE4idWxrCf7H8n976vYeX/xnb8z3D5vez76frG+Iwd9hn2yxDZxPQXKIPIkO3ZfmC4c+fiuq2DdO3LHnrsnfFblMSiCdNIc3Pzg0qpF1SrOh9F0dE+5m4sK0N6UR/25W9qajokiqImrXUTgEOUUguttTl/s+y31u5QSnUbYzq6u7vbff/KLHomRNA0QqPFIi5YsOCQBQsWLDLGHABgHpKLpdVa91prt3d2dm7OlMVJqTqPgjCHqPqdaGpqOoyZlxHR0QCe5ZPAFgJo8N/v3czcS0RPM/MfrbUPEdEjnZ2dWzKrUZhaq0dh4kht0QkiFsA9g9P2vZRE/LFW7qGOt++abx/bcjDtHiwEB+XBdbkwnWtJ8pidlN3CGRFjOjo6tgKIXw9OcD0qcwGyc+B4c4WVsjz2Xbt2bfXFmseCKkTf3hB+1eIvkRnDvmI0y9BsveGPZcmaznkc63jNxrmp9p2IrwVPA4hfP57k+miGLH5jbW8yxp3pGNNU3KYzdex5CnMwk+OZtYgAnCKUxvlxUuSFiKGVweBg3nT2HITtffO1IkYhF0WkNMHYzEdr6ySbY3CFaMteSMa7ic0WF+9UqRz7WPud3ee9LXJn64V6rh37vTXe2Xq8JkJ2jqjiesAV+6Yyy1W+vze/I/tivmfbuT+Xz7m9hgjAyeLsfM7c56x+RIBWFlGkbPeWg3nL9gXKWk35wKSLO2/hHofZCvuKWr6Q7O2SKoIwmxnvWjDbRJAgjIkIwAlCaRcPYvIFXqC0dfF/23c02M1bFtJgKacDzdAqgkUSf6bAUMTQcg8VBEEQBGGWIAJwHMiXDOJE+CGx+jHpnLX9fTnbs3kh7dpdp7Vi5HPGd3kD6eRjRE4AWpCSh0NBEARBEGYHIgDHgDipV+pSfS2xawuhLcKSMj1dC3nHjnmKmSgRfs7+76pplcvBOAHJ7hdFYgEUBEEQBGF2IAKwCr53ry/ll7h7tbJgC96+dZ7d1jOPwkhrrdkHBSa9rbUXfhgWPc8i/gRBEARBmFWIABxO0rwDLreDnQJUDILlgV05u3XzPBrcndNKMXI5A+Y0zm+k8EtJBaAWF7AgCIIgCLMEEYAp3uaHchcPC6UswkFld2xu4L4deQUQ5TJxforYC0aM2vkwXlJTEgcoCIIgCIIwG6h5AZi6e112r3X9Q6A1wxpwb0/B9vYUlI2Ucu5eFxQ4PM4P41SbTGMASbKABUEQBEGYJdSsAKSMu9fb/Jy7F8Tcv0Pzzu48Sv1JnJ/KWSf84LqZT0z4DW0ncQErcQELgiAIgjBLqEkBWLbeeXcvyJJSzGE/2b7uHPfvCLRSQOrutc7di3HdvdW3xSBKXsLMwcw2fTl7rqvCk2j0UqkkBY0FQRAEIUOtCUBvvfNdPGKFEDA4hNnVFXD/Fq3YQAU56xcm5+6tlt07UZwFUEknkJmGiOqISPlX+jf3M5fLZVs0iQgUBEEQap5aEYBM/oczDbkuHuzMRoNbifu7NaJB0u5vOWTcvTxl4ZeSuIDjn+ICnmGeZOYCM0cAdNYCaK0t+mVE/AmCIAhCLQhAH+tHlhmKwaTI+XCjXWx2dyoq7SSlFBDkXIKHz+4tW/wm5e6tvv3E/Su1AGeW9vb21WPo9HAvD0cQBEEQZjX7rwD0FiDLLreDFRFIA1y0dqCLUdzi2vRS4u51ii+N8+PJxvmNRTkJRCyAM42IPEEQBEGYIPudACRXvtkHgFnr2reRTjr6ljYbO9jFypZAWmOorEvi9N1jd2/V8aQuYCUWQEEQBEEQZgf7lwD0nXvjn+TdvbGaM73WljoiRLugSQF6WJzfzAi/lEQAWkkCEQRhv4NdI6SJLUoSg5vM19oqt5p1sDI/wt5mvxCAlAb8e6ufy/7UAA9YG7aHMFsNxSJP53wqCIOUmlpZl0mPDUkGsJYYQEEQxiGNPUbyv1kfNzKZMWbiqmf9fk037GwNTuRZrBt1GboYoEtqcH6EfcNcF4DO3WuJSFnrOnqQJrBhNh0ljrpCQgiigNxjarnH7wxb/bJIFrAgCBPFW4HmxNOiu4a24CQoNMDGl92hy6nOtj4ibIfFE7QB3ZkY6zmxj9MBZ0Qvvw4vsRYvVcCzwGgAoReEPwK4h27E7/38qFoUycLeZ04KwKR9m7t+uCi+NLuXCWy3hmzbi7C7jVKagFx8JbKJu3cP6vlNfayud5yrBSgIgjAafBoWo4BrwchBAyWDDxbW45FZK5iWI2DC9ymPJTDO6VId6y6+vXwGfm0MvkhtuGXvDnTfkR67qBVvUAr/BMZyVXnXjY+sQWjPwJ0mwkW0AfdVisD039yKtyOH82LBzREeVW14697fK2F/Ye5FpnEq4iiJ9Yu/XgGBB401f9jN/IfdRAOR0jk40UVkSRNz/CKKdaLPyt2Lr6QTiDzQCYIwkvShFBr1UDgNOZyCAKeoCIf4RfbW8+rk2OSkSy8iGA4RjiVR2eIAKLxa53GzbcG/Y3Lxg3OSVPzZFnxe57GOCMvZ+t7zIxfOkcJpOsA9ZiX+nxN72eM+FDfYhADHQuNFAI7dazsj7JfMGQtgIvdA7j+bmP+QU0DRMNoHrO0cVGRAyJFv98DpU/Net/oNG7fvNqxm4QO8IAj7nrJ1z8CyxQBZ5J00IkR+kdl58VjuLqka7JLriA3uJ+AvrhA70osf6gA8nwIcxpFz2YRUjw9xC7qpDZ+dtdbNPSS12JkWvI/qcT4PInR3J3LH+VZr8UswNkPhIAJeSoRVMMjF92Qq4DthC/5MbbhnxPwwQkSwMG5++/fpTgpznjkhAImYjU00oMulCBScCNw8wPhrP6HfaBWk7l7eZ+7eqmN3LmBIFrAgCFUp3+TzXlCN9KYGlXUueS30aJmjWcvaeLFk2aSTiSw/2koQQHERl6kN+N6It1dhHgivg8KVABZwEREpXMivxnX0Ezw9WRFYMeYR2cXV9j+eL1rnZNPY610LVblcur7JzI2z4J2CA5hwMUowUFBE6EGItbQBGyuXD1fi+CCPHwJYQgStCZ8B8IryAuuG9pGpvH9qIsc63a94HeX5SLORx8g+rjbPfnvjZnRXW/9EtjnKuG25f3/m30i/B+n+j3F8/bjj82z4sU2+R1yrMZezWwByYi1nS6RgXTaF1QTVW7Tq6T5ge0lRfFgLynmD48OcEX4zmt07Uchf3kmygAVBmCjxpU+XH2Cd+ONT0YADQbQOfenNrlrCwCSFyp4nnXiXCwj17ka7HBqbyjdappudpeo7UQt26gA3wSBCAfW2gDcCuNyHIo0pziYz5mr7X56vFixAAUVah1L2/fI8psvFolVBow19kxUHZetfHU7UARZxhBLlkLclXKg3YCOvQIAlmfGvA9Et+A234EM0DzcgAiiPl/NqHE0b8HA2iSS+oWh3S0R8RzETEPjD9mvYHK2rWGbkPA6b52HJLPH87IKijeir2J4T89WynSeyzbHG7cYzdN4n2xlH1FeOG4nom48dYLoDu6lifeOta39jFgtA15QNsbRz/9MaGAg5+Gsfq67dzrA25O61aRePGS/rMlnIP0eJC1gQhAnBThJRYJObllmD9yjC29ni2QiheA26mXBnsR9foh/jz1mrDLfidGi8lyMUYwFDJXyIbsVfKm9w6Q02Wo1X6Tw+yCEiKOTDCB8t3IxHp3BDdBVYneXrOSDaNPzmzoCmNrTZFjxMGi+ABZPCCckHYbgFh4PwA7+4LfZjbd0deLJiHWkixJkc4AwybnwP0034eDreWOAx4SoC5rsP9eM9CNCLenwSFq1QWMIll237ysx6nUDg1+BwrsdZBLySGc2usNga9DLwgCF8L7gR6yc0E4mVK759PduXfyBEYEW4x4m/jb4cTMW+bQXuOHgAXwWQpyA7sS6O8NuUw5EcYbENywfmcG7Fj9w2CEEY4cL8BjyQPXbxdoqtWJZXeD1bvJSAJUwuPqoTjHtRxA10G54YbZ6hscb9McLt1IareBVORx4fYItltBg5bsGZ1Ibbh52D8e9nYDWA1zDjKAIWMDsXeDsBP49lHbWhY7RzrHw8VuI5yOPvYPF/mHAwGH2kcB+K+CrdiseLq3BEPodz/EyWsBufpTvRO/LEBPMqvAw5vBOMEzjEIswD2zXoAeFXFOHrtAH3TejY7mfMPgF4Sfk3l+5htSKyzLpjF+undxINGuVi/zJxfjRL3L3VSAUgqZq0MAuCMFl8udJYvtgWXE55nAfjn2yT22UjBTi2rgHv4pU4i27BjZy4jU1YwtO5BqyiXCwjXCbBPQA+P5qVTRH+AXm0kAJsCVvzRTyNbFzi5MY8Fun6HoPC0UhKxizNvDsPOZfYkCiAAPWotMx4YQXgOCpgtYuQLOIwAB8v2yEtCtB4s/t/BER1uDRQuAwBXunsqMk4+zKDSsXOP0DjMlKYHwsKovIamwAcpQlv4jOwHiWcSbdg+5gCeV15Tvr9WiwCkB3EMXojHuTlyPGm4ZZK93sbdgF4zyjzewLyONyVPfNHUSkciBzO8C545HbjagAPpMc6Pie4Ff8GjXNjcU86eYeG7HqtIFzILfgcteGS8j5l5jk+N9zJEOFJsxpvQQHfdfOD8p4d4JZcDk2bEPIqvBg5XAOF43yqZnL/G9rm62FxIa/Gx2gDvjbagwm34L0IcBkUFrgxq/KxWwGFv+fVWBsytqKQfDfcuZDDlwAn2CmjCYjX4AoonOuPZ7KuZGPNLpEmh7NtKy5X6/HRWrMEzr7ItDRHypVwUUb3DprCQ92c/+M2pSNDquDcqZRk91qOf8Yv121jFr5c9m/8e+2cU4Ig7AHuXmmcBe98CpIbHEeI2KCXGcYVuS8hAuNgLuCGcDVWkL/h52/Dg1zCXU7sFWEUcAa8lS27DXeTPR4LQVjB/e72GS97Nd2B3TxGRZc9hWjo/jvMCmZhXXJDBMsGEUzy3rCb8VAcXD9CJEsBOzPTBoTuc9tdZrLBFk24FHm8csgh7f6vONEk5BM1zkMBX2KDhnQ5NniaQzzMUWJRchbSHFo5hzZeFkvrMUkstxHug/FxnQZW5XBZaRVeHAulbCwer3VJM+T/rd1rbcUxYPwWJTzABn9S5C0fFiEX0c0hOlF01qzd5cXXOvF3A9XhPEQI4hVzhBKKeJRL+AOHTizH66hHPS62LfhGWTQNuW773RwPIiLC0aRwZSr0ywdFeS2YiL8jkccdsfjj+Diwm7cdXMTDXMKfOPKPIBYHow7/xa04IxNXOCT+VuNsFPBlNpjvlwdKeJKLeMQOYicCHACNH+QUjscgBhEm3w1/7NPxkbeIf5sKONedU0k2ejwPT3CIx9gkYQBsYKgOH+HVuCJrTKoFZp8AJN+ira+UxxM9uv6hbuidRaICQen4sTERfvuyrMvkXsnlhmbfTAuCMAvxoicHhb8DOyHzabI4lhSeRyUcYyNcTMrdwGMhQYHCNXwaChhy+10Pgmbj3CMv4VU4AplyM+XEgUacTAEWlfONS7jeD2HG3BXMOMraxF3MQGf5jcD9TSFJcFDu39kSORhWCkWVk2WoLJSS9+qHMpMBHEgBTkWEPg7xaRisgMVxxC72ULmb/Wl4kdL4HBcRUhJz+TsYnEq7cZRajxcMMpYhwoXxuLiIIuXxf+3hOD8rXCpJRUT+FjzEFj+kAgJrnP3xGbkcfmHX4EpuxUtjIem6g6xzsXyp8OBYrFfGtlEb3ko34IVkcZ7b62Rvf0/AkRSPsYjnYwd+6UVULNo+TnVo5QEMUg6KI3yDLP4WebyA/oAXEOElsLiLAigeRJHq8E5uwTnu3FtWnlNVPo/gzpVDUMIdNsSbKMJyhPg/iPDf7jgtQ54DrEOAg53Y0ghtiPMHBrCM1uMYKmEZGbyMGQ+zgnEZ78BneTlyacmb+OfgaXguB7giFtz+eGxChJPQjaPj9SiLY3gAn4HGgaxwObOrRRmfLQEK5fPDdf83q3AWFfBWHsQgBQhg8aN43PS4G9PRbj4svkEaARdRQgEf4NV41VjHdn9j9rmA/UUs/z+dlzBQT/V4LcWq3XAUn4w0C92840LJt1IQBGECUNnFZnCWWo9vZt7bDOASbsEjHOAHiBBRHs+HRSvBx9D1oY0X4gsA5lEOOWuwEsB/VLqBGXgDEZg0FBv8Rt2GBzEV9y+854adVU3jYV8Gxr+TtkDjVpyBAEdxiJAC5DhxT88MPvcOwC5YvFa14d7Kt93PHC6iJEOXmfEQDeKVdFvZqoh5begA8C/cil4EuAIlWCKcy6twhU9uGRMaxNms8AxVhxO46P5UoBzOhcG5/Dz8kY/APZbwM1XCf9OtroTOaJ1ShlJthnycEbVhx4hdPx2HQuNjKCKiHOpi8afW412ZRQwewSZegdfiQNxL2ok5w4SLuQXXoS2T2JE0WTWxgGKDr6n1OKvafu5qxIL5wINcRI4CHI4SzlEbMuftbYj3/lfhSrw3KGAjRy4G9HlowlHYhAewAhobEeXzOI80CrFEZIsnaTtOoY2ZfbwZTwH4uFmNkqrDRVxCpBlshk9UxCegnjUu5Ph4BahDhO/Terxl2KDb8BiAd9lWF3P5/3zi1UcA/HTWll6aZmadyk1P/PzNuF/djNNg8SYGHqc6CkgTQVHEilxELRRx+vvsfSXPmnbWzbQgCLOQWIhYBNAc4Q5aj2/yCgSpy9K9ViCgNpcPeYuzbLh2SHgDkrtWQD9FNyzupCC+dbsb7ev8ulO3qt16GhaCcSqHiZ2HLK7zn5+8+5fTFBCXMWvoEZQoyVA13rqjola8gRW+DptYdriEXYpH64o7TSRz+BVaj3td3J0vm8LeQtS/Es1QeHUsRkDQtoR/isWfs2ZllnWJG+vxn2zwuEsM0WgylCSwjGUFdD9/jG20zcUffpYUdlAOyh2FRMQdjgDvUAG+iTwe5jNwIyfu/GriIy0WTmlxW1DmnIAX38l+vwE5zHdFMQy29/UlsW3ZsbpzaGP8+ICPOeumgaUcllqL04dt37UxRY4NtlAPPpx+litK0Cy8C1vVTXgr7cALEOJobMC3eC3yla7soA+/ZINel9+p3bwf5jazERGvcDUjV7vgBg1NFhfH4i8+duVx++2qJ/BpDvE0KWhTxSZkluBVFOAw91BisAuDOM99NnsexL+vhaYIFzqbqXHzeiKvwqJaiQOcjRZARzmp4wb8YPMK3HrIYj6fFJ1PAS2wJba+rorGUI2h2Wlj40QAqtShIQiCMDqU/p8Y33LXtY3D66bxxrK78LtQWOltei/kjIXPAt/VhDWInMXqeF6J59At+JMTiEB0cB6xrFns4u5C9GMAN6BKrOCERxxvR+EDthWnl61VbrCYzwrLtMJRbNy4LRWgeRD/qDag24uxyW9zPNhnCTL+283VpqGSKalwqQ9wvOu+YlysWo++Bbe73XlkeImYtGqfZfwMAZ4Xr0KT68Lx00zCxGjDINqIQQD/zCvxn8jj9cxYTcBxUDjQW3njiWkgjTXIYY1dgyvpJnxotHp7lInDq0igSH8/yUXgBVCIcFcszipLoTjxB2ct/gUHeJIUnuW2p3ESkIQCKALb5C4bC8kH6R7s8vsTVd3XpKZePM+PuT9kSu3waSgM5rHEWqxRSe9o59EznNEgC/E8EJ7hdivETljc6ra3aagGZvpAER8jPhw/QR7vrtaDRjFWILFuE0f4Df3Yhxtks9Pj3ze53/5kWvCYCvBCYvdg9HwAW2ohIWTWCsBs8Uhfa+iS4hr+Xo7wKZVXb4rfsxEiTuJAiGarEPQC0Or9+jwSBGF6YBcHFzmH1IM+HqmSpDBuCb9PHILumncIVuIAugXbkTwZ384hNhNhMQIUrHVlOa7AMig84i5La4mcSIhvkHepO0cvyzEuSUICU4AToXBixd4kLwtQzhUsUSjiE7oNV8XX9otn6gZLfiaTOMnhc5iKNsYRrsRY5MqoNHArbrNUjqMc2oWklr9h4Hmupl8AxYRG9+a6scc/7D52C9oBfDF+8WosBeNlTC7m7GQKsMwLwYjq8H5uQZ1qw3smc0zKApfwXLIuOpIt8D9+n1VlLcBUzNk1eBgaz07SNnB4+r7l8tkVC+nisJmttv20pt5r0Ih6vJyB48FOMDcycHABWEoFNMTykYGIkuOTXdffuLi/JDnlKXUztlTdUX/8LPCIonIVkOEovx/s0nCOtq1O3DtXf+Wi/g/PYIsSJWJ36Xj7ur8wawVgSkUl98cBfnP0ZvNNRfpSVUcvjk8mY4eEoHOH0CwSgV4AsppUrVNBEGoT1gQ2FmHRJrFPlQKg/G/lsoJDUi4rtYCiK52y3dfc22VbcDNyONPV3CNXz+0KZzk5EQsYeLWz42iQNfj20KqnFP8H72oDVbnEuZs90AGLjdEgvpS7Fb/OWKSoYuHRt58VWxMZJfk0kaF/8bD1KNT7sVvSmIc6nDraatxP6/ux5N3Pxe6PY1gAKztpDOtksQHdAG6KX7wCgTkQq5XCFUQ4DIMoUg5/H7bgO9SGu6eQkDDf7yEpRs+wfR65a7G42+runElyzsLs+Ckp9UIYuqeOOvO8AnV8ED7DhHeQwkFp2Znyp5Js9vicnufkKQ1lEfvRHODmKdnWiNjGMtlscAaNctYe4ErAJA8mhyKPQ0dbnRuA8a+8S4ZqyL61PzPrBWBK/HRRdgt/H7fzCnOnbQrOJYWP6zwtNiV/ihG0mm3WQKKh6tQXDat1KAiCkIWsT6aoC4Zin7KUrULava+9iIlQN9x1aQ2+pxnvYuNuky9L3cDmIJykAyyFcZ/q0ru863MPsn8p5yyJ3wDhS76n7VAHB3b17Z5KkyayXSBS86WzeAb+ds7D6uMN3dqHLHfs+wxP7fqeXoP9GkhBMWMz+nHHWCtkcoUdLIrIEeNu98d1o89Z1Y4lmc4TSAQh0TonxW/k0/EoF/BrAPVE7kHgTYDbzuT2k11Rb/ebTXoxjydUC8Tlun5Rxfgrtz5Cbvl9UTgAP6ICTktthVzENgL+F4RHAVds+iGK8CRr3E1URZCpxNUbj4UJ80bdv3RfyDVP5KoPDezOKHhr+iMc4X/IG4iqrpOg/HmXixh/9H/d74v3zhkBiJFu4QiI/qN/LdbVqdyFKqCzXEBoiIiTL7SaFW7hsgXQn3ci/gRBGAt2sUsaRRcP9ccqbbPSm3CTW47drasXnUnNujSm7kmLnz8nwp8V4ZmuSIZ3AyvtEkbYdxi+iTaib7zWXOOM11lyGPizuslHVVVfbERPXUr3JS37kkoOhWxnp9Ryl34oUJz4wafkoLukfD/o9iJBw6JDrcfbJruqsdyz3IplINQ7OdyPLtyCjkyHjkQcDrVGC+hW/N624DeUw8m+qf2z/KosLgK5e4fxx22s/SZ0g1wcGxHhOe5vVSyA5ePAeGZZejO60vddDOA4+1/uKNOCVl3AaTyIIjQKFOEKCvFZunVofW75E1CPpdXL4kYG3UF8HiU37GfyaViYzcgu4/dFMQ73mecjp4DcdpkCJyt/rtbjnHF2ZeQ69nP3L2ZjFvBEyLqF561Du/p2eLZh+3LL2KjrKNCaFKicLUxpUb59lQmcZAHPyakWBGFvwu4GyL7K3aIk8DoAABJmSURBVMn+r8MfYIfq4b0CQx0SHnPFeDO1/o64DUVi3ODEVRIas9JnVL7GXUEtKDJJ9u8eQeWEi8KITEtfdNkvVq0FWnKTteW/m/imnVcubox8oh8uymawAif6WoJTxW0rMnjA+YySOLQj+XQ80409yXLV/pUUaq4szDwBGPgOAvwWwG+hcY2LVVsx0ujCw61qC9PfXPu0BBpmOBiygA4rWpzJ4L7fpwPFy500ythcVjCvxlJSOAYWxnXcIPyuPEkTs7AmdXsJr0p6siIPg/tovWtB2OW2szbJpHb7vgALnAu4irgqlfA4G/QTO5F7iOveEi+YnE/Dali69RJe6/dx5DgtHnB/Ny6rfoUbQ7IePeLYzhZP4T5gTquS1C0cH8jcd6J79bdKJ1mDM63CX4L6+DLifK8GXoRRWpl5b780uaE4LtrXsyYIwqwl6fJAHLnSLmfyiVjgEhBSUbICgbvurUAdNM50Wbzaicbb/BqyUVcghe85F3DSkeKFaMZ7iLAUSXuv3wc345fYQ/evH3e5FzCekwi9zGtUSwoNZQt3M1wZmcSio/A+3+bTuSQvSdZjzGqcTRrHsRnKDJ38UJNtBga/Y4s/u3HnUMc5vNdlnG507dASEhetK8zMLfgir8Gl/HqscbX2RiFj6fwtkmSdiDVO5tV4SZpBWxZg3sLpjvEqrCaN5a5DhXLz+VA6hmwZGL8RSrrhJ0WL0168brsRbnQPEtaJ6WN5lSvtYt05lIrZFS772jLhXOQwn5NOLAgtNkxyOtNjO9+nL8XS9M/ujWXIu5hHuGzjJHu4gFdSvD0afvzic7shydT9ja/tYVjjUnf+J51TUsupO6dsCz5KAY6I91FXO78sfuy+Q8Yd2yNRxEp3bFc4tzvcawWUS1xZgUP4DFxr1uBj3ILX8qox3M/7GXNaAMJ/mdPU8Pjf+triN3cViy/iiP9NBTQY1JFmIsvKtQ6JiX8keV178SV1AAVBmADJzd4AlEczFuHbfCoa4punq6u3EdGTsfg7CNdC4ZmcCIxeNYjvIuP+LQurm/A7WDzoIwUPZsJnmF2nBmLC99MWcvtyh50ISjI+H3b2sQghNF7PrbiKV+EIbsWBvAbP41Z8RgW4uhyZtgd2G+e6vA1FAr6IvHMTFknhfG7F29NOHNmOHLwGF6EO70eAjyPAjdDOxYoxEzQY13qhZl2SToCbzRq8JRYYNFQgm3kFDuQWvA85fK/sAo98RxeU4wzdO8Y6Nz85wabw3NJpOCYrtp016xZs5BC/csWbLSLO4b+4BS9y55AXs67u3mq8ngJ8jEOUKO9q/d2a34AHqlkpxyCx7jKeimfCtVVTOIlX49muHmS6vU0I47FygMud1a7Surjci2aLy104gXH1Io/mxfiZF2WLuAULiqfjKG7FvymNy/wjC9mKM8FnXD8Ui0DKu6zekDW+yqtxwog5OA0FPhBXIY+/UzlcxuQEcM0IwDkVAzgWZbfwRVB0CbYDxX/kd+a+Yyl3aa6A1fG7YYSIk9ZBe61sDKd1ALUoQEEQRlIWayUw8i6AP4DFEyjiUcxDKzPu4xZcbxl/hUKzIrwJCss4QkgF5FDEP9Id2HwRoC7BsH6orsaeAX5ACn/rL0ULnIs5RKhCfN8vOjXr3yYwnuGD8DmTwTtOaZTqU+AE0ZehcQLi63QEQzmcw8C7AWwF4yAqoMAl7AJjEym8wrluh4L6k58DYBQyozEV7w9tMBFLf8CX+Ai8ifJ4KZdcO7hv2Va8nRl3xNtVwBIorILGiVxEP+UxjwewXm3AxsraepXrpjbcY1fjapqHc3gAIRSWKI3vMuMpV3olKZx9IAPHUA6HclLPzlAdcjyAy1UbHszEQLrx6wFsYoVtpHBgLCqDPH5qW11SRYASPkq34nG3sxrnsMWvCW42DmWNe3gNvmsZ96mk3d5JILyRjdvnHAy2k8EH3A70la2MI4/tSGtb4k5nbAgsLqAkEXNxvD3b6upYPhZvjxkvIYW3Ij7/DIqudVsSu8r+XHLePNqAm7kF11I93hHPmetSEuA2tu4cGMzlsBgF5Lnoqv/9kBTeXKVOUiJKI3wQGptAzjoZj+luuwY/ZIt7wRhQCn/juuFoLONBDFAB9cT4RPwwUgs1ALE/CcAUuiRTNuab4YNA2FJ8d/3rAk2fys1TR0clhnsqosT3P9NCkNI6gDUbZSAIwlhksnqVsz4ETgzWoR9ngXA7NeBFiHCxGsrUTD6UQ44H8a+qDdeMksSR3JxDrMsTLiZAu7IxAQIOca+6FY/v0Y1uuRtNUO7gm1oSxymOXElZMK3Ht20rVlId3oSSK85siJAD4dDUwUkW5wJ4NnI4ydnmqHwPS+Lo6kG+BHLaVVgNe79y24+gxM1ogcJ6KuCEeLsU4FQinIrsfBuACpiHEL+i4rC2aqPtk4vPu3gD3ndRK4jyODsthUIKf4P4ReV9co5uShJhAh7AlaoNH6k8Nu7fd6LXtODDpHGtq62osAgKr0MesBY3A3jchQnciPuj1ThD5/AdaBxMBvUI8G6VCOqExMqcQ4R2lPBGXyjcFc32S2jv0kVGKwybx3Q/aQPus6vxJZqH9/GgG3gjcvjHNEODvI2ZB/ApKFcm5pk+frUcslDuh9yLs1i5UjjvSeff9SEeGrcldi3pdiCHt1DJfY5RLLuJEzF5Kx6PVmGNzuN60jgEEXII8BYC3pIeW7J+/QXUo4hr0YbP1Yr4w/7gAh6NTHwgFb428CNV7D8uCu0nlMaOXD0FrJwb2LAimum2ckhbwgmCIFRAQ4kQJQAPIMLjBDyJO9CDAZyCQVyNCB0cwXAJlg12gPFThFip2nDBGJYod0Mt3IrHwfiZFxgufYIJ3/GLTf3KtMmNezsYO2DRy4Td7u+TtwCW58D1ay3i4yD8qRz3xtgNi3tRxOupzVmVCi7r2bp56PWrSLYZunnYFo+J42W4XBqnWhZsIl7uwOY/PoiTUMIFTPhL2eFKwxbuRITP4CmcTD/GNkwgSzR+/xLAqvU4xxRxOhtsACefrTKiHja4EQanqLbEElet/mM8Xt2Gb6GEVjB+yYz++LyI95sIR7oFNyYhUcEG3IYQxyHC11zGs8GQLTEp9tLOIa5EP15Ct+Be/xDBmQ3udsc1cTv3jjePagPOxSA+RQo7ykEF6RxG+CuK+Hva4NJZQnfskvWWKtdFGxGpm3A2IrwSBl9lxn0I8Sgb/A4G10YlvCo+DwAcVK3u71CrZKjgZtxVLOJ4NvgWCNvL3xLKzAXhMRRxDq3HO2mU7iv7KzVhlxpWd+qswrNNLvcppeht8VlSitxXwXfsnX5rIIGjXJ6CUpGvK1zT9zbvot7v6wsJgjB5MhmzPMxD0epcfoc5q0yIrrS8xnjWivTaZ1twJeVxLofOGbGrr4QjFtzmiwTvyXhX4qA+i2A+3Mh2p7X+9ni9p6GAOjwHjHmI0EM34ymk83Oqs5ImcVohSnRnWZwk76/Bwa7RmAFhM3ZkW4lV3VZ2nldhHjSWA66Ey8Fg9EHhUezCb7LbmdS+DF//IuRxOAwOBaPOMAZ0Dh3YjSdSYTmBYzq0vtfhmQidm3oQu/FUxVwM3fdW4iAEOAqERud2zaEdIR6hNlejseo2+VQ0lOcZKFYtyVJtfK9BIxpwIgye6+yHscAq4W66xRUpVzgZBzlLbXx8/oLebOu9NJllxFgqLNy81vVuPlPl8RVOPt1PEZ5Pt6A9uy/ZsAhejaVGYbkGDgehAYQtsHgYf8BvR7T/qxFqQgCm+G4izrwdnd1wMgJ9qc7RS03IiFzhSPLxgeyFIO3x/MQCMJ+noFjk6+qu3ikCUBCECZMWDK5SPqXq3zPvK5flmDTZn88H4QkwFrmCzSF+oNbjzXtU+28GGW1cMz3e8foS78n2RxM2FcuMeUynMpYJnSfTaPUaa1yTmb9SC16UC/FX3IYtLmM5zZ5OElQC2ohBuxpfoAI+jNDdsLdThMPTVogV2x13Xmfrd2Gm2e9iAMci002E6JrddzHw0+jche+FogsKeTq0OJgkWjGUpmxMwp6iFEhJGzhBECZHNgEgk3HKVbtMDP+cxUZ/QzsQ/0oBlnDooqQKZPD16Rxj1mo5HevLdAmhTIII0/AEl1G3mb1mT2ZMqfhz87w2c91fV87YnXqnlJH7RMOHPP4xHWd9VUVclfOn3NJtIvs02WNbTq5J5jChYv6qHZ/Uahetxmkq54o2r2aNO2kZVtHGIeucF4IRr8RzOMDbXbKQdsXenqwm/rJzkO2+Un5zGo7tXKamBCCyJ0NqifvPnV/uO6vhBqbggiBHZ2tFwWCJDSddEKfHLayS4l6CIAhTZTI3KduKL8Q3RjY4EhqvjcUf5VwW7a/oZtwxWtzgFMc1IzFTYwmiCdUWnPp27WSSWCa57gmLvOle31SO91Tm0n/GjDaHo6wzEbGEF1I9WrgPRQpwKj8P9/IR+CxC3AeDXajHIst4JSt8koBFbDGIPOpgXZzjmJa8yu4rQg0KwJRh2cL/tbsbwLn971/4rRzUv9bVq1OtAUqGI+b4MjqU+TQlIagJVsrACIIwg5SD31fj2VD4MLS/qybFcAsw2BoS3l2YLs+GIEwTmdI5n7UteD7V4Z1cdFnOy6GwjoHdlMcAAwtVzhVLik9iQ/WoQxG3YTu+Op0PNbVCzauScqHPi6DmXbnzN7n/2PHqMLRvs4Qn6upVQEnBVEOuqYjr6MHp7xN9xaeldIITBGGGcaLOAie6hm9RWRUW2eAO9GNFYT0eqaUyF8LcIT0nVRvOxCD+GQpbXDaxciWPGpDDIgqQd0sF7u8WRVyFIs7whdLlnJ4k8hSYIZsxtPX9WNhQOORjivDhXEDzBoppX2yalFuYwFFdQQUDg+a6ef++TZJABEGYUfgUHIAFOBYRDnSlLwL8lX7kyqqMm2EqCLMFPhVLMB8r2eLlBDyXCQ2uhLlCBwP3KYs2Wo9H9vU45zIiAKuQFWnFD80/ShXq/iVQ9Pr4sjkQcZItPEG3cCwA6wsq6B+01zVcvkUEoCAI+wQRf8JcYRJZzjWZvTtdiGOyCtn4wMJ/9P0+d9mWN5QiXh2B7q+fpwMdEEEhSgo8E5Eijl/wRZ+rvrRobUEQ9g7xjZHhuh2p6c7SFYSZJo0JjO/B7pUxsrhzem25k5eIvz1ABOAYDOsm8rktNz/+xObjSyF/FAG2xULQ+rbt43UTca3g9vXOCIJQM/jSFsb/FOEnzDlc6Z91MO6VOYfdOV3xN2FqiAAch3I9wIugjl6HUuEzmz9vB8JjS6H9ej6naF6dip9EDJLWcpRoweGJIkkSiEy1IAiCIAizA1ElEyTrFq7/wvanC5dufrfhaEWJ8YuGBq2DgBQrRFYRrHLpwpxaBJ0FUO/rPRAEQRAEQUgQAThJym7hi6DqL91yd+FTXS/vD/k9UPhrQ0MQJHVihruFoV35mH09dEEQBEEQBIcIwCng3MKXwMYiMP53w6c6v1os9h0blvgLuRyV5s/T2rW6TqyAlCSBKIlXEARBEARhViACcA/IuoUXfmbX1vzF7edH4BNCg1sb5mmdz5NictnCbJ0KFARBEARB2PeIAJwGst1EGi7s+J/8BX9dORDZN4Lo0YZ5uTpoRdDJMrhEMpcEQRAEQdi3iACcRpxb2JeNmffJ9nXbtpWOG4zMBVbTIBMOdAtdJMW3BUEQBEEQ9kvS+MCYbZ96xjG7Lz3s7/btiARBEARBEIS9wg/WQgrACIIgCIIg1BrMSdmYfT0OQRAEQRAEQRAEQRAEoQYRq5QgCIIgCEKNIQJQEARBEAShxhABKAiCIAiCUGOIABQEQRAEQagxRAAKgiAIgiDUGCIABUEQBEEQagwRgIIgCIIgCDWGCEBBEARBEIQaQwSgIAiCIAhCjSECUBAEQRAEocYQASgIgiAIglBjiAAUBEEQBEGoMUQACoIgCIIg1BgiAAVBEARBEGoMEYCCIAiCIAg1hghAQRAEQRCEGkMEoCAIgiAIQo0hAlAQBEEQBKHGEAEoCIIgCIJQY4gAFARBEARBqDFEAAqCIAiCINQYIgAFQRAEQRBqDBGAgiAIgiAINYYIQEEQBEEQhBpDBKAgCIIgCEKNIQJQEARBEAShxhABKAiCIAiCUGOIABQEQRAEQagxRAAKgiAIgiDUGCIABUEQBEEQagwRgIIgCIIgCDWGCEBBEARBEIQaQwSgIAiCIAhCjSECUBAEQRAEocYQASgIgiAIglBjiAAUBEEQBEGoMUQACoIgCIIg1BgiAAVBEARBEGqM/x8AAP//gP3LzOmb+xsAAAAASUVORK5CYII= + mediatype: image/png + description: | + Portworx-Enterprise is the most widely-used and reliable cloud-native + storage solution for production workloads and provides high-availability, + data protection and security for containerized applications. + + Portworx Enterprise enables you to migrate entire applications, including + data, between clusters in a single data center or cloud, or between clouds, + with a single kubectl command. + + The cloud native storage and data management platform that enterprises trust + to manage data in containers now has an operator which simplifies the install, + configuration, upgrades and manages the Portworx Enterprise cluster lifecycle. + + Learn more about the Portworx Enterprise + [the data platform for Kubernetes](https://portworx.com/products/introduction) + + To learn more about the platform features, please visit our + [product features page](https://portworx.com/products/features) + + ### About Portworx + + Portworx is the solution for running stateful containers in production, + designed with DevOps in mind. With Portworx, users can manage any database + or stateful service on any infrastructure using any container scheduler, + including Kubernetes, Mesosphere DC/OS, and Docker Swarm. Portworx solves + the five most common problems DevOps teams encounter when running stateful + services in production: persistence, high availability, data automation, + security, and support for multiple data stores and infrastructure. + + ### How to install StorageCluster + + To customize your cluster's configuration (specification), use the + [Spec Generator](https://central.portworx.com/) from PX-Central. + + ### Prerequisite + + Ensure ports 17001-17020 on worker nodes are reachable from master and other worker nodes. + + ### Tutorials + + * [Portworx Enterprise on Openshift](https://portworx.com/openshift) + + * [Stateful applications on Kubernetes](https://docs.portworx.com/portworx-install-with-kubernetes/application-install-with-kubernetes) + + * [Portworx Enterprise on Kubernetes](https://docs.portworx.com/portworx-install-with-kubernetes) + + * [Kafka on Kubernetes](https://portworx.com/kafka-kubernetes) + + * [Elastisearch on Kubernetes](https://portworx.com/elasticsearch-kubernetes) + + * [PostgreSQL on Kubernetes](https://portworx.com/postgres-kubernetes/) + + * [MongoDB on Kubernetes](https://portworx.com/mongodb-kubernetes/) + + * [Cassandra on Kubernetes](https://portworx.com/cassandra-kubernetes/) + + * [Kubernetes backup and recovery](https://portworx.com/kubernetes-backup/) + + * [Disaster Recovery for Kubernetes](https://portworx.com/kubernetes-disaster-recovery/) + + ### Uninstall + + Deleting the StorageCluster object for Portworx cluster does not stop Portworx + service running on the nodes, to avoid application downtime. + + To uninstall Portworx completely without wiping the data, you should add the + following delete strategy to the StorageCluster spec: + ``` + spec: + deleteStrategy: + type: Uninstall + ``` + **Caution:** To uninstall Portworx and **wipe all the data**, you should use the following + delete strategy: + ``` + spec: + deleteStrategy: + type: UninstallAndWipe + ``` + + installModes: + - type: OwnNamespace + supported: true + - type: SingleNamespace + supported: true + - type: MultiNamespace + supported: true + - type: AllNamespaces + supported: true + install: + spec: + clusterPermissions: + - serviceAccountName: portworx-operator + rules: + - apiGroups: + - "*" + resources: + - "*" + verbs: + - "*" + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - anyuid + - privileged + - portworx + verbs: + - use + deployments: + - name: portworx-operator + spec: + replicas: 1 + selector: + matchLabels: + name: portworx-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + name: portworx-operator + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - portworx-operator + topologyKey: kubernetes.io/hostname + containers: + - name: portworx-operator + image: portworx/px-operator:24.1.2-drw-dev + imagePullPolicy: Always + command: + - /operator + - --driver=portworx + - --leader-elect=true + env: + - name: OPERATOR_NAME + value: portworx-operator + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + serviceAccountName: portworx-operator + strategy: deployment + customresourcedefinitions: + owned: + - kind: StorageCluster + name: storageclusters.core.libopenstorage.org + version: v1 + displayName: Storage Cluster + description: Storage Cluster installs Portworx in the cluster. It has all the necessary configurations to setup and update a Portworx cluster. + specDescriptors: + - description: Details of the storage used by the storage driver. + displayName: Storage + path: storage + - description: List of devices to be used by the storage driver. + displayName: Device list + path: storage.devices + - description: Details of storage used in cloud environment. + displayName: Cloud Storage + path: cloudStorage + - description: >- + List of storage device specs. A cloud storage device will be + created for every spec in the list. + displayName: Device spec list + path: cloudStorage.deviceSpecs + - description: Maximum nodes that can have storage in the cluster. + displayName: Max storage nodes + path: cloudStorage.maxStorageNodes + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:advanced' + - description: Maximum nodes in every zone that can have storage in the cluster. + displayName: Max storage nodes per zone + path: cloudStorage.maxStorageNodesPerZone + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:advanced' + - description: >- + Maximum nodes in every zone in every node group that can have + storage in the cluster. + displayName: Max storage nodes per zone per node group + path: cloudStorage.maxStorageNodesPerZonePerNodeGroup + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:advanced' + - description: The docker image name and version of Portworx Enterprise. + displayName: Image + path: image + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:advanced' + - 'urn:alm:descriptor:com.tectonic.ui:text' + - description: >- + CustomImageRegistry is a custom container registry server (may + include repository) that will be used instead of index.docker.io + to download Docker images. (Example: myregistry.net:5443 or + myregistry.com/myrepository) + displayName: Custom Image Registry + path: customImageRegistry + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:advanced' + - 'urn:alm:descriptor:com.tectonic.ui:text' + - description: >- + It is a reference to a secret in the same namespace as the + StorageCluster. This secret is used to pull images from a private + registry. + displayName: Private Registry Image Pull Secret + path: imagePullSecret + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:advanced' + - 'urn:alm:descriptor:io.kubernetes:Secret' + - description: Contains security configuration for the storage cluster. + displayName: Role Based Access Control + path: security + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:advanced' + - description: >- + Flag indicating whether security features need to be enabled for + the storage cluster. + displayName: Enabled + path: security.enabled + - description: >- + The secrets provider which will contain secrets that are needed by + Portworx for features like volume encryption, cloudsnaps, etc. + displayName: Encryption Provider + path: secretsProvider + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:advanced' + - 'urn:alm:descriptor:com.tectonic.ui:select:k8s' + - 'urn:alm:descriptor:com.tectonic.ui:select:vault' + - 'urn:alm:descriptor:com.tectonic.ui:select:aws-kms' + - 'urn:alm:descriptor:com.tectonic.ui:select:azure-kv' + - 'urn:alm:descriptor:com.tectonic.ui:select:ibm-kp' + - description: List of environment variables used by the storage pods. + displayName: Environment variables + path: env + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:advanced' + - description: >- + It is the pull policy for the image. Accepts one of Always, Never, + IfNotPresent. Defaults to Always. + displayName: Image Pull Policy + path: imagePullPolicy + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Use all available, unformatted, unpartitioned devices. + displayName: Use all available devices + path: storage.useAll + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Use all available unformatted devices. + displayName: Use all available unformatted devices + path: storage.useAllWithPartitions + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: >- + Flag indicating to use the devices even if there is file system + present on it. Note that the devices may be wiped before using. + displayName: Force use devices + path: storage.forceUseDisks + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Device used for journaling. + displayName: Journal device + path: storage.journalDevice + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Device that will be used to store system metadata by the driver. + displayName: System metadata device + path: storage.systemMetadataDevice + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Device used for internal KVDB. + displayName: KVDB device + path: storage.kvdbDevice + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Device used for caching. + displayName: Cache devices + path: storage.cacheDevices + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Device spec for the journal device. + displayName: Journal device spec + path: cloudStorage.journalDeviceSpec + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Device spec for the metadata device. + displayName: System metadata device spec + path: cloudStorage.systemMetadataDeviceSpec + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Device spec for internal KVDB device. + displayName: KVDB device spec + path: cloudStorage.kvdbDeviceSpec + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Capacity specs. + displayName: Capacity specs + path: cloudStorage.capacitySpecs + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Node pool label. + displayName: Node pool label + path: cloudStorage.nodePoolLabel + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Cloud provider + displayName: Cloud provider + path: cloudStorage.provider + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: The network configuration used by storage nodes + displayName: Network Configuration + path: network + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: KVDB configuration + displayName: KVDB configuration + path: kvdb + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Resources configuration + displayName: Resources configuration + path: resources + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Autopilot configuration + displayName: Autopilot configuration + path: autopilot + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Stork configuration + displayName: Stork configuration + path: stork + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: UI configuration + displayName: UI configuration + path: userInterface + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Contains monitoring configuration for the storage cluster. + displayName: Monitoring + path: monitoring + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Auth configuration for RBAC + displayName: Auth configuration + path: security.auth + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Volumes configuration + displayName: Volumes configuration + path: volumes + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Nodes configuration + displayName: Nodes configuration + path: nodes + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Delete strategy + displayName: Delete strategy + path: deleteStrategy + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Update strategy + displayName: Update strategy + path: updateStrategy + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Auto update strategy for components + displayName: Component update strategy + path: autoUpdateComponents + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Describes placement configuration for the storage cluster pods. + displayName: Placement + path: placement + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: It is the starting port in the range of ports used by Portworx. + displayName: Start Port + path: startPort + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: >- + Version is a read-only field. It contains the current version of + Portworx. + displayName: Version + path: version + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + - description: Revision history limit is the number of old histories to retain. + displayName: Revision History Limit + path: revisionHistoryLimit + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:hidden' + statusDescriptors: + - path: conditions + displayName: Cluster Conditions + description: Conditions describe the current state of the cluster + x-descriptors: + - 'urn:alm:descriptor:io.kubernetes.conditions' + - path: phase + displayName: Status + description: Status of the Portworx cluster. + x-descriptors: + - 'urn:alm:descriptor:io.kubernetes.phase' + resources: + - kind: Pod + name: "" + version: v1 + - kind: Service + name: "" + version: v1 + - kind: Deployment + name: "" + version: v1 + - kind: DaemonSet + name: "" + version: v1 + - kind: ConfigMap + name: "" + version: v1 + - kind: StorageNode + name: storagenodes.core.libopenstorage.org + version: v1 + displayName: Storage Node + description: Do not create Storage Node as it is internally created by the operator. It represents the status of a Portworx node. + specDescriptors: + - path: version + displayName: Version + description: Version of Portworx on the node. + statusDescriptors: + - path: nodeUid + displayName: Node UID + description: Unique identifier for the Portworx node. + - path: phase + displayName: Status + description: Status of the Portworx node. + x-descriptors: + - 'urn:alm:descriptor:io.kubernetes.phase' + - path: network.dataIP + displayName: Data IP + description: IP address used by the storage driver for data traffic + - path: network.mgmtIP + displayName: Management IP + description: IP address used by the storage driver for management traffic + - path: conditions + displayName: Node Conditions + description: Conditions describe the current state of the storage node + x-descriptors: + - 'urn:alm:descriptor:io.kubernetes.conditions' diff --git a/deploy/olm-catalog/portworx/portworx.package.yaml b/deploy/olm-catalog/portworx/portworx.package.yaml index 63a746fec3..6bb4d9843f 100644 --- a/deploy/olm-catalog/portworx/portworx.package.yaml +++ b/deploy/olm-catalog/portworx/portworx.package.yaml @@ -1,7 +1,7 @@ packageName: portworx-certified channels: - name: alpha - currentCSV: portworx-operator.v24.2.0 + currentCSV: portworx-operator.v24.1.2-drw - name: stable - currentCSV: portworx-operator.v24.2.0 + currentCSV: portworx-operator.v24.1.2-drw defaultChannel: stable diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 789acae144..0986332e0a 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -20,7 +20,7 @@ spec: spec: containers: - name: portworx-operator - image: portworx/px-operator:24.2.0-dev + image: portworx/px-operator:24.1.2-drw-dev imagePullPolicy: Always command: - /operator diff --git a/drivers/storage/portworx/component/csi.go b/drivers/storage/portworx/component/csi.go index 671564812c..25fb48c55e 100644 --- a/drivers/storage/portworx/component/csi.go +++ b/drivers/storage/portworx/component/csi.go @@ -22,6 +22,7 @@ import ( storagev1beta1 "k8s.io/api/storage/v1beta1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -441,56 +442,6 @@ func (c *csi) createDeployment( return err } - var ( - existingProvisionerImage = k8sutil.GetImageFromDeployment(existingDeployment, csiProvisionerContainerName) - existingAttacherImage = k8sutil.GetImageFromDeployment(existingDeployment, csiAttacherContainerName) - existingSnapshotterImage = k8sutil.GetImageFromDeployment(existingDeployment, csiSnapshotterContainerName) - existingResizerImage = k8sutil.GetImageFromDeployment(existingDeployment, csiResizerContainerName) - existingSnapshotControllerImage = k8sutil.GetImageFromDeployment(existingDeployment, csiSnapshotControllerContainerName) - existingHealthMonitorControllerContainerName = k8sutil.GetImageFromDeployment(existingDeployment, csiHealthMonitorControllerContainerName) - provisionerImage string - attacherImage string - snapshotterImage string - resizerImage string - snapshotControllerImage string - healthMonitorControllerImage string - ) - - provisionerImage = util.GetImageURN( - cluster, - cluster.Status.DesiredImages.CSIProvisioner, - ) - if provisionerImage == "" { - return fmt.Errorf("csi provisioner image not found") - } - if csiConfig.IncludeAttacher { - attacherImage = util.GetImageURN( - cluster, - cluster.Status.DesiredImages.CSIAttacher, - ) - if attacherImage == "" { - return fmt.Errorf("csi attacher image not found") - } - } - if csiConfig.IncludeSnapshotter { - snapshotterImage = util.GetImageURN( - cluster, - cluster.Status.DesiredImages.CSISnapshotter, - ) - if snapshotterImage == "" { - return fmt.Errorf("csi snapshotter image not found") - } - } - if csiConfig.IncludeResizer { - resizerImage = util.GetImageURN( - cluster, - cluster.Status.DesiredImages.CSIResizer, - ) - if resizerImage == "" { - return fmt.Errorf("csi resizer image not found") - } - } - if cluster.Spec.CSI.InstallSnapshotController != nil && *cluster.Spec.CSI.InstallSnapshotController && cluster.Status.DesiredImages.CSISnapshotController != "" { @@ -501,26 +452,6 @@ func (c *csi) createDeployment( return err } } - if !*c.csiSnapshotControllerPreInstalled { - snapshotControllerImage = util.GetImageURN( - cluster, - cluster.Status.DesiredImages.CSISnapshotController, - ) - } - } else if cluster.Spec.CSI.InstallSnapshotController != nil && - *cluster.Spec.CSI.InstallSnapshotController && - cluster.Status.DesiredImages.CSISnapshotController == "" { - return fmt.Errorf("csi snapshot controller image not found") - } - - if csiConfig.IncludeHealthMonitorController { - healthMonitorControllerImage = util.GetImageURN( - cluster, - cluster.Status.DesiredImages.CSIHealthMonitorController, - ) - if cluster.Status.DesiredImages.CSIHealthMonitorController != "" { - return fmt.Errorf("csi health monitor controller image not found") - } } updatedTopologySpreadConstraints, err := util.GetTopologySpreadConstraints(c.k8sClient, csiDeploymentTemplateSelectorLabels) @@ -528,21 +459,15 @@ func (c *csi) createDeployment( return err } - deployment := getCSIDeploymentSpec(cluster, csiConfig, ownerRef, provisionerImage, attacherImage, - snapshotterImage, resizerImage, snapshotControllerImage, healthMonitorControllerImage, updatedTopologySpreadConstraints) - modified := provisionerImage != existingProvisionerImage || - attacherImage != existingAttacherImage || - snapshotterImage != existingSnapshotterImage || - resizerImage != existingResizerImage || - snapshotControllerImage != existingSnapshotControllerImage || - healthMonitorControllerImage != existingHealthMonitorControllerContainerName || - util.HasPullSecretChanged(cluster, existingDeployment.Spec.Template.Spec.ImagePullSecrets) || - util.HasNodeAffinityChanged(cluster, existingDeployment.Spec.Template.Spec.Affinity) || + deployment := c.getCSIDeploymentSpec(cluster, csiConfig, updatedTopologySpreadConstraints, existingDeployment.Annotations, ownerRef) + isPodTemplateEqual, _ := util.DeepEqualPodTemplate(&deployment.Spec.Template, &existingDeployment.Spec.Template) + + modified := !isPodTemplateEqual || util.HasSchedulerStateChanged(cluster, existingDeployment.Spec.Template.Spec.SchedulerName) || - util.HaveTolerationsChanged(cluster, existingDeployment.Spec.Template.Spec.Tolerations) || util.HaveTopologySpreadConstraintsChanged(updatedTopologySpreadConstraints, existingDeployment.Spec.Template.Spec.TopologySpreadConstraints) || - hasCSITopologyChanged(cluster, existingDeployment) + hasCSITopologyChanged(cluster, existingDeployment) || + !equality.Semantic.DeepEqual(deployment.Annotations, existingDeployment.Annotations) if !c.isCreated || modified { if err = k8sutil.CreateOrUpdateDeployment(c.k8sClient, deployment, ownerRef); err != nil { return err @@ -598,15 +523,12 @@ func hasCSITopologyChanged( return existingTopologyFeatureGate != updatedTopologyFeatureGate } -func getCSIDeploymentSpec( +func (c *csi) getCSIDeploymentSpec( cluster *corev1.StorageCluster, csiConfig *pxutil.CSIConfiguration, - ownerRef *metav1.OwnerReference, - provisionerImage, attacherImage string, - snapshotterImage, resizerImage string, - snapshotControllerImage string, - healthMonitorControllerImage string, topologySpreadConstraints []v1.TopologySpreadConstraint, + existingAnnotations map[string]string, + ownerRef *metav1.OwnerReference, ) *appsv1.Deployment { replicas := int32(3) leaderElectionType := "leases" @@ -617,6 +539,56 @@ func getCSIDeploymentSpec( } imagePullPolicy := pxutil.ImagePullPolicy(cluster) + var ( + provisionerImage string + attacherImage string + snapshotterImage string + resizerImage string + snapshotControllerImage string + healthMonitorControllerImage string + ) + + provisionerImage = util.GetImageURN( + cluster, + cluster.Status.DesiredImages.CSIProvisioner, + ) + if csiConfig.IncludeAttacher && cluster.Status.DesiredImages.CSIAttacher != "" { + attacherImage = util.GetImageURN( + cluster, + cluster.Status.DesiredImages.CSIAttacher, + ) + } + if csiConfig.IncludeSnapshotter && cluster.Status.DesiredImages.CSISnapshotter != "" { + snapshotterImage = util.GetImageURN( + cluster, + cluster.Status.DesiredImages.CSISnapshotter, + ) + } + if csiConfig.IncludeResizer && cluster.Status.DesiredImages.CSIResizer != "" { + resizerImage = util.GetImageURN( + cluster, + cluster.Status.DesiredImages.CSIResizer, + ) + } + + if cluster.Spec.CSI.InstallSnapshotController != nil && + *cluster.Spec.CSI.InstallSnapshotController && + cluster.Status.DesiredImages.CSISnapshotController != "" && + !*c.csiSnapshotControllerPreInstalled { + snapshotControllerImage = util.GetImageURN( + cluster, + cluster.Status.DesiredImages.CSISnapshotController, + ) + + } + + if csiConfig.IncludeHealthMonitorController && cluster.Status.DesiredImages.CSIHealthMonitorController != "" { + healthMonitorControllerImage = util.GetImageURN( + cluster, + cluster.Status.DesiredImages.CSIHealthMonitorController, + ) + } + var args []string if util.GetImageMajorVersion(provisionerImage) >= 2 { args = []string{ @@ -645,10 +617,13 @@ func getCSIDeploymentSpec( Privileged: boolPtr(true), } + annotations := util.GetCombinedAnnotations(cluster, k8sutil.Deployment, CSIApplicationName, existingAnnotations) + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: CSIApplicationName, Namespace: cluster.Namespace, + Annotations: annotations, OwnerReferences: []metav1.OwnerReference{*ownerRef}, }, Spec: appsv1.DeploymentSpec{ @@ -658,7 +633,8 @@ func getCSIDeploymentSpec( }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: csiDeploymentTemplateLabels, + Labels: csiDeploymentTemplateLabels, + Annotations: annotations, }, Spec: v1.PodSpec{ ServiceAccountName: CSIServiceAccountName, @@ -699,6 +675,11 @@ func getCSIDeploymentSpec( }, } + if cluster.Spec.CSI.ExternalProvisioner != nil && + cluster.Spec.CSI.ExternalProvisioner.Resources != nil { + deployment.Spec.Template.Spec.Containers[0].Resources = *cluster.Spec.CSI.ExternalProvisioner.Resources + } + if pxutil.IsStorkEnabled(cluster) { deployment.Spec.Template.Spec.SchedulerName = util.StorkSchedulerName } @@ -763,6 +744,10 @@ func getCSIDeploymentSpec( "--leader-election-type=configmaps", ) } + if cluster.Spec.CSI.Snapshotter != nil && + cluster.Spec.CSI.Snapshotter.Resources != nil { + snapshotterContainer.Resources = *cluster.Spec.CSI.Snapshotter.Resources + } deployment.Spec.Template.Spec.Containers = append( deployment.Spec.Template.Spec.Containers, snapshotterContainer, @@ -770,47 +755,57 @@ func getCSIDeploymentSpec( } if csiConfig.IncludeResizer && resizerImage != "" { - deployment.Spec.Template.Spec.Containers = append( - deployment.Spec.Template.Spec.Containers, - v1.Container{ - Name: csiResizerContainerName, - Image: resizerImage, - ImagePullPolicy: imagePullPolicy, - Args: []string{ - "--v=3", - "--csi-address=$(ADDRESS)", - "--leader-election=true", - }, - Env: []v1.EnvVar{ - { - Name: "ADDRESS", - Value: "/csi/csi.sock", - }, + resizerContainer := v1.Container{ + Name: csiResizerContainerName, + Image: resizerImage, + ImagePullPolicy: imagePullPolicy, + Args: []string{ + "--v=3", + "--csi-address=$(ADDRESS)", + "--leader-election=true", + }, + Env: []v1.EnvVar{ + { + Name: "ADDRESS", + Value: "/csi/csi.sock", }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "socket-dir", - MountPath: "/csi", - }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "socket-dir", + MountPath: "/csi", }, - SecurityContext: sc, }, + SecurityContext: sc, + } + if cluster.Spec.CSI.Resizer != nil && + cluster.Spec.CSI.Resizer.Resources != nil { + resizerContainer.Resources = *cluster.Spec.CSI.Resizer.Resources + } + deployment.Spec.Template.Spec.Containers = append( + deployment.Spec.Template.Spec.Containers, + resizerContainer, ) } if csiConfig.IncludeSnapshotController && snapshotControllerImage != "" { + controllerContainer := v1.Container{ + Name: csiSnapshotControllerContainerName, + Image: snapshotControllerImage, + ImagePullPolicy: imagePullPolicy, + Args: []string{ + "--v=3", + "--leader-election=true", + }, + SecurityContext: sc, + } + if cluster.Spec.CSI.SnapshotController != nil && + cluster.Spec.CSI.SnapshotController.Resources != nil { + controllerContainer.Resources = *cluster.Spec.CSI.SnapshotController.Resources + } deployment.Spec.Template.Spec.Containers = append( deployment.Spec.Template.Spec.Containers, - v1.Container{ - Name: csiSnapshotControllerContainerName, - Image: snapshotControllerImage, - ImagePullPolicy: imagePullPolicy, - Args: []string{ - "--v=3", - "--leader-election=true", - }, - SecurityContext: sc, - }, + controllerContainer, ) } @@ -860,21 +855,28 @@ func getCSIDeploymentSpec( ) } - if cluster.Spec.Placement != nil { - if cluster.Spec.Placement.NodeAffinity != nil { - deployment.Spec.Template.Spec.Affinity = &v1.Affinity{ - NodeAffinity: cluster.Spec.Placement.NodeAffinity.DeepCopy(), - } - } + var nodeAffinity *v1.NodeAffinity + var tolerations []v1.Toleration + if cluster.Spec.CSI.Placement != nil { + nodeAffinity = cluster.Spec.CSI.Placement.NodeAffinity + tolerations = cluster.Spec.CSI.Placement.Tolerations + } else if cluster.Spec.Placement != nil { + nodeAffinity = cluster.Spec.Placement.NodeAffinity + tolerations = cluster.Spec.Placement.Tolerations + } - if len(cluster.Spec.Placement.Tolerations) > 0 { - deployment.Spec.Template.Spec.Tolerations = make([]v1.Toleration, 0) - for _, toleration := range cluster.Spec.Placement.Tolerations { - deployment.Spec.Template.Spec.Tolerations = append( - deployment.Spec.Template.Spec.Tolerations, - *(toleration.DeepCopy()), - ) - } + if nodeAffinity != nil { + deployment.Spec.Template.Spec.Affinity = &v1.Affinity{ + NodeAffinity: nodeAffinity.DeepCopy(), + } + } + if len(tolerations) > 0 { + deployment.Spec.Template.Spec.Tolerations = make([]v1.Toleration, 0) + for _, toleration := range tolerations { + deployment.Spec.Template.Spec.Tolerations = append( + deployment.Spec.Template.Spec.Tolerations, + *(toleration.DeepCopy()), + ) } } diff --git a/drivers/storage/portworx/component/portworx_api.go b/drivers/storage/portworx/component/portworx_api.go index e4b4859650..24d95b3003 100644 --- a/drivers/storage/portworx/component/portworx_api.go +++ b/drivers/storage/portworx/component/portworx_api.go @@ -12,7 +12,9 @@ import ( "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -175,59 +177,12 @@ func (c *portworxAPI) createDaemonSet( return getErr } - var existingPauseImageName string - var csiNodeRegistrarImageName string - var existingCsiDriverRegistrarImageName string - if len(existingDaemonSet.Spec.Template.Spec.Containers) > 0 { - existingPauseImageName = existingDaemonSet.Spec.Template.Spec.Containers[0].Image - } - - pauseImageName := pxutil.ImageNamePause + daemonSet := getPortworxAPIDaemonSetSpec(cluster, existingDaemonSet.Annotations, ownerRef) + isPodTemplateEqual, _ := util.DeepEqualPodTemplate(&daemonSet.Spec.Template, &existingDaemonSet.Spec.Template) - if cluster.Status.DesiredImages != nil && cluster.Status.DesiredImages.Pause != "" { - pauseImageName = cluster.Status.DesiredImages.Pause - } - - // If px version is greater than 2.13 then update daemonset when csi-node-driver-registrar image changes if CSI is enabled - pxVersion := pxutil.GetPortworxVersion(cluster) - addOrRemoveCSIRegistrar := false - checkCSIDriverRegistrarChange := false - isCsiEnabled := pxutil.IsCSIEnabled(cluster) - pxVersionGTE2_13 := pxVersion.GreaterThanOrEqual(csiRegistrarAdditionPxVersion) - hasMultipleCsiContainers := len(existingDaemonSet.Spec.Template.Spec.Containers) > 1 - isSingleCsiContainer := len(existingDaemonSet.Spec.Template.Spec.Containers) == 1 - - if isCsiEnabled && pxVersionGTE2_13 && hasMultipleCsiContainers { - existingCsiDriverRegistrarImageName = existingDaemonSet.Spec.Template.Spec.Containers[1].Image - csiNodeRegistrarImageName = cluster.Status.DesiredImages.CSINodeDriverRegistrar - csiNodeRegistrarImageName = util.GetImageURN(cluster, csiNodeRegistrarImageName) - checkCSIDriverRegistrarChange = true - } else if !isCsiEnabled && pxVersionGTE2_13 && hasMultipleCsiContainers { - // When CSI is disabled, remove the csi-node-driver-registrar container from the daemonset - addOrRemoveCSIRegistrar = true - } else if !pxVersionGTE2_13 && hasMultipleCsiContainers { - // When px version is less than 2.13, remove the csi-node-driver-registrar container from the daemonset - addOrRemoveCSIRegistrar = true - } else if isCsiEnabled && pxVersionGTE2_13 && isSingleCsiContainer { - // When px version is greater than 2.13 and CSI is enabled, add the csi-node-driver-registrar container to the daemonset - addOrRemoveCSIRegistrar = true - } - - pauseImageName = util.GetImageURN(cluster, pauseImageName) - serviceAccount := pxutil.PortworxServiceAccountName(cluster) - existingServiceAccount := existingDaemonSet.Spec.Template.Spec.ServiceAccountName - - modified := existingPauseImageName != pauseImageName || addOrRemoveCSIRegistrar || - (checkCSIDriverRegistrarChange && (existingCsiDriverRegistrarImageName != csiNodeRegistrarImageName)) || - existingServiceAccount != serviceAccount || - util.HasPullSecretChanged(cluster, existingDaemonSet.Spec.Template.Spec.ImagePullSecrets) || - util.HasNodeAffinityChanged(cluster, existingDaemonSet.Spec.Template.Spec.Affinity) || - util.HaveTolerationsChanged(cluster, existingDaemonSet.Spec.Template.Spec.Tolerations) - if !c.isCreated || errors.IsNotFound(getErr) || modified { - daemonSet := getPortworxAPIDaemonSetSpec(cluster, ownerRef, pauseImageName) - if pxutil.IsCSIEnabled(cluster) && pxVersion.GreaterThanOrEqual(csiRegistrarAdditionPxVersion) && len(daemonSet.Spec.Template.Spec.Containers) < 2 { - c.recorder.Event(cluster, v1.EventTypeWarning, util.FailedComponentReason, "Failed to setup CSI. CSI driver registrar container image not found") - } + if !c.isCreated || errors.IsNotFound(getErr) || + !isPodTemplateEqual || + !equality.Semantic.DeepEqual(daemonSet.Annotations, existingDaemonSet.Annotations) { if err := k8sutil.CreateOrUpdateDaemonSet(c.k8sClient, daemonSet, ownerRef); err != nil { return err } @@ -238,16 +193,25 @@ func (c *portworxAPI) createDaemonSet( func getPortworxAPIDaemonSetSpec( cluster *corev1.StorageCluster, + existingAnnotations map[string]string, ownerRef *metav1.OwnerReference, - imageName string, ) *appsv1.DaemonSet { maxUnavailable := intstr.FromString("100%") startPort := pxutil.StartPort(cluster) + pauseImageName := pxutil.ImageNamePause + if cluster.Status.DesiredImages != nil && cluster.Status.DesiredImages.Pause != "" { + pauseImageName = cluster.Status.DesiredImages.Pause + } + pauseImageName = util.GetImageURN(cluster, pauseImageName) + + annotations := util.GetCombinedAnnotations(cluster, k8sutil.DaemonSet, PxAPIDaemonSetName, existingAnnotations) + newDaemonSet := &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: PxAPIDaemonSetName, Namespace: cluster.Namespace, + Annotations: annotations, OwnerReferences: []metav1.OwnerReference{*ownerRef}, }, Spec: appsv1.DaemonSetSpec{ @@ -262,7 +226,8 @@ func getPortworxAPIDaemonSetSpec( }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: getPortworxAPIServiceLabels(), + Labels: getPortworxAPIServiceLabels(), + Annotations: annotations, }, Spec: v1.PodSpec{ ServiceAccountName: pxutil.PortworxServiceAccountName(cluster), @@ -271,10 +236,13 @@ func getPortworxAPIDaemonSetSpec( Containers: []v1.Container{ { Name: "portworx-api", - Image: imageName, + Image: pauseImageName, ImagePullPolicy: pxutil.ImagePullPolicy(cluster), ReadinessProbe: &v1.Probe{ - PeriodSeconds: int32(10), + PeriodSeconds: int32(10), + TimeoutSeconds: int32(1), + SuccessThreshold: int32(1), + FailureThreshold: int32(3), ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Host: "127.0.0.1", @@ -290,6 +258,22 @@ func getPortworxAPIDaemonSetSpec( }, } + if cluster.Spec.PortworxAPI != nil && + cluster.Spec.PortworxAPI.Resources != nil { + newDaemonSet.Spec.Template.Spec.Containers[0].Resources = *cluster.Spec.PortworxAPI.Resources + } else { + newDaemonSet.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("50Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("200m"), + }, + } + } + // If CSI is enabled then run the csi-node-driver-registrar pods in the same daemonset // Do this only if portworx version is greater than 2.13 pxVersion := pxutil.GetPortworxVersion(cluster) @@ -331,6 +315,7 @@ func getPortworxAPIDaemonSetSpec( } } } + newDaemonSet.Spec.Template.ObjectMeta = k8sutil.AddManagedByOperatorLabel(newDaemonSet.Spec.Template.ObjectMeta) return newDaemonSet @@ -381,7 +366,8 @@ func csiRegistrarContainer(cluster *corev1.StorageCluster) *v1.Container { Name: "KUBE_NODE_NAME", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", + APIVersion: "v1", + FieldPath: "spec.nodeName", }, }, }, @@ -426,6 +412,24 @@ func csiRegistrarContainer(cluster *corev1.StorageCluster) *v1.Container { if container.Name == "" { return nil } + + if cluster.Spec.CSI != nil && + cluster.Spec.CSI.NodeDriverRegistrar != nil && + cluster.Spec.CSI.NodeDriverRegistrar.Resources != nil { + container.Resources = *cluster.Spec.CSI.NodeDriverRegistrar.Resources + } else { + container.Resources = v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("50Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("200m"), + }, + } + } + return &container } diff --git a/drivers/storage/portworx/components_test.go b/drivers/storage/portworx/components_test.go index 3d3f4e7092..4c97c71cd7 100644 --- a/drivers/storage/portworx/components_test.go +++ b/drivers/storage/portworx/components_test.go @@ -16,6 +16,7 @@ import ( "github.com/golang-jwt/jwt/v4" "github.com/golang/mock/gomock" + osdapi "github.com/libopenstorage/openstorage/api" ocpconfig "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" ocp_secv1 "github.com/openshift/api/security/v1" @@ -49,7 +50,6 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "sigs.k8s.io/controller-runtime/pkg/client" - osdapi "github.com/libopenstorage/openstorage/api" "github.com/libopenstorage/operator/drivers/storage/portworx/component" "github.com/libopenstorage/operator/drivers/storage/portworx/manifest" pxutil "github.com/libopenstorage/operator/drivers/storage/portworx/util" @@ -664,7 +664,233 @@ func TestPortworxAPIDaemonSetAlwaysDeploys(t *testing.T) { require.NoError(t, err) require.NotEqual(t, "new/image", ds.Spec.Template.Spec.Containers[0].Image) require.NotEqual(t, "new/csi-driver-image", ds.Spec.Template.Spec.Containers[1].Image) +} + +func TestPortworxAPIResourcesChange(t *testing.T) { + coreops.SetInstance(coreops.New(fakek8sclient.NewSimpleClientset())) + reregisterComponents() + k8sClient := testutil.FakeK8sClient() + driver := portworx{} + err := driver.Init(k8sClient, runtime.NewScheme(), record.NewFakeRecorder(10)) + require.NoError(t, err) + + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-cluster", + Namespace: "kube-test", + }, + Spec: corev1.StorageClusterSpec{ + Image: "portworx/oci-monitor:3.2.0", + }, + } + createTelemetrySecret(t, k8sClient, cluster.Namespace) + err = driver.SetDefaultsOnStorageCluster(cluster) + require.NoError(t, err) + + // TestCase: Both API and CSI registrar containers should have default resources. + defaultResources := v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("50Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("200m"), + }, + } + err = driver.PreInstall(cluster) + require.NoError(t, err) + + ds := &appsv1.DaemonSet{} + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Len(t, ds.Spec.Template.Spec.Containers, 2) + require.Equal(t, defaultResources, ds.Spec.Template.Spec.Containers[0].Resources) + require.Equal(t, defaultResources, ds.Spec.Template.Spec.Containers[1].Resources) + + // TestCase: Add resources to API container + cluster.Spec.PortworxAPI = &corev1.PortworxAPISpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("200Mi"), + v1.ResourceCPU: resource.MustParse("200m"), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("400Mi"), + v1.ResourceCPU: resource.MustParse("400m"), + }, + }, + } + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, *cluster.Spec.PortworxAPI.Resources, ds.Spec.Template.Spec.Containers[0].Resources) + require.Equal(t, defaultResources, ds.Spec.Template.Spec.Containers[1].Resources) + + // TestCase: Add resources to CSI registrar container + cluster.Spec.CSI.NodeDriverRegistrar = &corev1.CSINodeDriverRegistrarSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("200Mi"), + v1.ResourceCPU: resource.MustParse("200m"), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("400Mi"), + v1.ResourceCPU: resource.MustParse("400m"), + }, + }, + } + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, *cluster.Spec.PortworxAPI.Resources, ds.Spec.Template.Spec.Containers[0].Resources) + require.Equal(t, *cluster.Spec.CSI.NodeDriverRegistrar.Resources, ds.Spec.Template.Spec.Containers[1].Resources) + + // TestCase: Update resources for both containers + cluster.Spec.PortworxAPI.Resources.Requests[v1.ResourceMemory] = resource.MustParse("300Mi") + cluster.Spec.CSI.NodeDriverRegistrar.Resources.Requests[v1.ResourceMemory] = resource.MustParse("300Mi") + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, *cluster.Spec.PortworxAPI.Resources, ds.Spec.Template.Spec.Containers[0].Resources) + require.Equal(t, *cluster.Spec.CSI.NodeDriverRegistrar.Resources, ds.Spec.Template.Spec.Containers[1].Resources) + + // TestCase: Removing resources should fallback to default + cluster.Spec.PortworxAPI.Resources = nil + cluster.Spec.CSI.NodeDriverRegistrar.Resources = nil + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, defaultResources, ds.Spec.Template.Spec.Containers[0].Resources) + require.Equal(t, defaultResources, ds.Spec.Template.Spec.Containers[1].Resources) +} + +func TestPortworxAPIAnnotationsChange(t *testing.T) { + coreops.SetInstance(coreops.New(fakek8sclient.NewSimpleClientset())) + reregisterComponents() + k8sClient := testutil.FakeK8sClient() + driver := portworx{} + err := driver.Init(k8sClient, runtime.NewScheme(), record.NewFakeRecorder(10)) + require.NoError(t, err) + + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-cluster", + Namespace: "kube-test", + }, + Spec: corev1.StorageClusterSpec{ + Image: "portworx/oci-monitor:3.2.0", + }, + } + createTelemetrySecret(t, k8sClient, cluster.Namespace) + err = driver.SetDefaultsOnStorageCluster(cluster) + require.NoError(t, err) + + // TestCase: API pods and daemonset should not have any annotations as it is not defined in the cluster spec. + err = driver.PreInstall(cluster) + require.NoError(t, err) + + ds := &appsv1.DaemonSet{} + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Empty(t, ds.Annotations) + require.Empty(t, ds.Spec.Template.Annotations) + + // TestCase: Define annotations for the API daemonset + apiDaemonsetAnnotationKey := fmt.Sprintf("%s/%s", k8sutil.DaemonSet, component.PxAPIDaemonSetName) + annotations := map[string]string{ + "key1": "value1", + } + cluster.Spec.Metadata = &corev1.Metadata{ + Annotations: map[string]map[string]string{ + apiDaemonsetAnnotationKey: annotations, + }, + } + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, ds.Annotations, ds.Spec.Template.Annotations) + require.Len(t, ds.Annotations, 2) + require.Contains(t, ds.Annotations, constants.AnnotationCustomAnnotations) + require.Equal(t, "value1", ds.Annotations["key1"]) + + // TestCase: Add annotations to the API daemonset + annotations["key2"] = "value2" + cluster.Spec.Metadata.Annotations[apiDaemonsetAnnotationKey] = annotations + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, ds.Annotations, ds.Spec.Template.Annotations) + require.Len(t, ds.Annotations, 3) + require.Contains(t, ds.Annotations, constants.AnnotationCustomAnnotations) + require.Equal(t, "value1", ds.Annotations["key1"]) + require.Equal(t, "value2", ds.Annotations["key2"]) + + // TestCase: Change annotation value for the API daemonset + annotations["key2"] = "value2-changed" + cluster.Spec.Metadata.Annotations[apiDaemonsetAnnotationKey] = annotations + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, ds.Annotations, ds.Spec.Template.Annotations) + require.Len(t, ds.Annotations, 3) + require.Contains(t, ds.Annotations, constants.AnnotationCustomAnnotations) + require.Equal(t, "value1", ds.Annotations["key1"]) + require.Equal(t, "value2-changed", ds.Annotations["key2"]) + + // TestCase: Remove an annotation key from the API daemonset. + // Also ensure that we don't remove any annotations that we did not add. + ds.Annotations["external-key"] = "external-value" + err = testutil.Update(k8sClient, ds) + require.NoError(t, err) + + delete(annotations, "key2") + cluster.Spec.Metadata.Annotations[apiDaemonsetAnnotationKey] = annotations + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, ds.Annotations, ds.Spec.Template.Annotations) + require.Len(t, ds.Annotations, 3) + require.Contains(t, ds.Annotations, constants.AnnotationCustomAnnotations) + require.Equal(t, "value1", ds.Annotations["key1"]) + require.Equal(t, "external-value", ds.Annotations["external-key"]) + + // TestCase: Remove all annotations from the API daemonset. + cluster.Spec.Metadata.Annotations[apiDaemonsetAnnotationKey] = nil + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, ds, component.PxAPIDaemonSetName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, ds.Annotations, ds.Spec.Template.Annotations) + require.Len(t, ds.Annotations, 1) + require.Equal(t, "external-value", ds.Annotations["external-key"]) } func TestPortworxProxyIsNotDeployedWhenClusterInKubeSystem(t *testing.T) { @@ -8053,6 +8279,427 @@ func TestCSI_0_3_NodeAffinityChange(t *testing.T) { require.Nil(t, csiStatefulSet.Spec.Template.Spec.Affinity) } +func TestCSIExtResourcesChange(t *testing.T) { + versionClient := fakek8sclient.NewSimpleClientset() + coreops.SetInstance(coreops.New(versionClient)) + versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ + GitVersion: "v1.29.0", + } + fakeExtClient := fakeextclient.NewSimpleClientset() + apiextensionsops.SetInstance(apiextensionsops.New(fakeExtClient)) + + reregisterComponents() + k8sClient := testutil.FakeK8sClient() + driver := portworx{} + err := driver.Init(k8sClient, runtime.NewScheme(), record.NewFakeRecorder(10)) + require.NoError(t, err) + + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-cluster", + Namespace: "kube-test", + }, + Spec: corev1.StorageClusterSpec{ + Image: "portworx/image:3.2.0", + }, + } + createTelemetrySecret(t, k8sClient, cluster.Namespace) + err = driver.SetDefaultsOnStorageCluster(cluster) + require.NoError(t, err) + + // TestCase: No resources defined in the cluster spec + err = driver.PreInstall(cluster) + require.NoError(t, err) + + csiDeployment := &appsv1.Deployment{} + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Len(t, csiDeployment.Spec.Template.Spec.Containers, 4) + for _, container := range csiDeployment.Spec.Template.Spec.Containers { + require.Empty(t, container.Resources) + } + + // TestCase: Define resources in the cluster spec + cluster.Spec.CSI.ExternalProvisioner = &corev1.CSIExternalProvisionerSpec{ + Resources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + } + cluster.Spec.CSI.Snapshotter = &corev1.CSISnapshotterSpec{ + Resources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + }, + }, + } + cluster.Spec.CSI.Resizer = &corev1.CSIResizerSpec{ + Resources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + }, + }, + } + cluster.Spec.CSI.SnapshotController = &corev1.CSISnapshotControllerSpec{ + Resources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("400m"), + }, + }, + } + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, *cluster.Spec.CSI.ExternalProvisioner.Resources, csiDeployment.Spec.Template.Spec.Containers[0].Resources) + require.Equal(t, *cluster.Spec.CSI.Snapshotter.Resources, csiDeployment.Spec.Template.Spec.Containers[1].Resources) + require.Equal(t, *cluster.Spec.CSI.Resizer.Resources, csiDeployment.Spec.Template.Spec.Containers[2].Resources) + require.Equal(t, *cluster.Spec.CSI.SnapshotController.Resources, csiDeployment.Spec.Template.Spec.Containers[3].Resources) + + // TestCase: Change resources in the cluster spec + cluster.Spec.CSI.ExternalProvisioner.Resources.Requests = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + } + cluster.Spec.CSI.Snapshotter.Resources.Requests = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + } + cluster.Spec.CSI.Resizer.Resources.Requests = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + } + cluster.Spec.CSI.SnapshotController.Resources.Limits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("400m"), + } + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, *cluster.Spec.CSI.ExternalProvisioner.Resources, csiDeployment.Spec.Template.Spec.Containers[0].Resources) + require.Equal(t, *cluster.Spec.CSI.Snapshotter.Resources, csiDeployment.Spec.Template.Spec.Containers[1].Resources) + require.Equal(t, *cluster.Spec.CSI.Resizer.Resources, csiDeployment.Spec.Template.Spec.Containers[2].Resources) + require.Equal(t, *cluster.Spec.CSI.SnapshotController.Resources, csiDeployment.Spec.Template.Spec.Containers[3].Resources) + + // TestCase: Remove resources from the cluster spec + cluster.Spec.CSI.ExternalProvisioner = nil + cluster.Spec.CSI.Snapshotter = nil + cluster.Spec.CSI.Resizer = nil + cluster.Spec.CSI.SnapshotController = nil + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + for _, container := range csiDeployment.Spec.Template.Spec.Containers { + require.Empty(t, container.Resources) + } +} + +func TestCSIExtSpecificNodeAffinityChange(t *testing.T) { + versionClient := fakek8sclient.NewSimpleClientset() + coreops.SetInstance(coreops.New(versionClient)) + versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ + GitVersion: "v1.29.0", + } + fakeExtClient := fakeextclient.NewSimpleClientset() + apiextensionsops.SetInstance(apiextensionsops.New(fakeExtClient)) + + reregisterComponents() + k8sClient := testutil.FakeK8sClient() + driver := portworx{} + err := driver.Init(k8sClient, runtime.NewScheme(), record.NewFakeRecorder(10)) + require.NoError(t, err) + + nodeAffinity := &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "px/csi", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"false"}, + }, + }, + }, + }, + }, + } + + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-cluster", + Namespace: "kube-test", + }, + Spec: corev1.StorageClusterSpec{ + Image: "portworx/image:3.2.0", + }, + } + createTelemetrySecret(t, k8sClient, cluster.Namespace) + err = driver.SetDefaultsOnStorageCluster(cluster) + require.NoError(t, err) + + // TestCase: CSI pods should not have any affinity it is not defined in the cluster spec. + cluster.Spec.Placement.NodeAffinity = nil + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + csiDeployment := &appsv1.Deployment{} + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Nil(t, csiDeployment.Spec.Template.Spec.Affinity) + + // TestCase: Define node affinity in the CSI spec + cluster.Spec.CSI.Placement = &corev1.PlacementSpec{ + NodeAffinity: nodeAffinity.DeepCopy(), + } + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, nodeAffinity, csiDeployment.Spec.Template.Spec.Affinity.NodeAffinity) + + // TestCase: Update node affinity in the CSI spec + nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution. + NodeSelectorTerms[0]. + MatchExpressions[0]. + Key = "portworx/csi" + cluster.Spec.CSI.Placement.NodeAffinity = nodeAffinity.DeepCopy() + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, nodeAffinity, csiDeployment.Spec.Template.Spec.Affinity.NodeAffinity) + + // TestCase: Remove node affinity from the CSI spec + cluster.Spec.CSI.Placement.NodeAffinity = nil + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Nil(t, csiDeployment.Spec.Template.Spec.Affinity) +} + +func TestCSIExtSpecificTolerationsChange(t *testing.T) { + versionClient := fakek8sclient.NewSimpleClientset() + coreops.SetInstance(coreops.New(versionClient)) + versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ + GitVersion: "v1.29.0", + } + fakeExtClient := fakeextclient.NewSimpleClientset() + apiextensionsops.SetInstance(apiextensionsops.New(fakeExtClient)) + + reregisterComponents() + k8sClient := testutil.FakeK8sClient() + driver := portworx{} + err := driver.Init(k8sClient, runtime.NewScheme(), record.NewFakeRecorder(10)) + require.NoError(t, err) + + tolerations := []v1.Toleration{ + { + Key: "px/infra", + Value: "csi", + Operator: v1.TolerationOpEqual, + Effect: v1.TaintEffectNoSchedule, + }, + } + + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-cluster", + Namespace: "kube-test", + }, + Spec: corev1.StorageClusterSpec{ + Image: "portworx/image:3.2.0", + }, + } + createTelemetrySecret(t, k8sClient, cluster.Namespace) + err = driver.SetDefaultsOnStorageCluster(cluster) + require.NoError(t, err) + + // TestCase: CSI pods should not have any tolerations as it is not defined in the cluster spec. + cluster.Spec.Placement.Tolerations = nil + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + csiDeployment := &appsv1.Deployment{} + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Empty(t, csiDeployment.Spec.Template.Spec.Tolerations) + + // TestCase: Define tolerations in the CSI spec + cluster.Spec.CSI.Placement = &corev1.PlacementSpec{ + Tolerations: tolerations, + } + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, tolerations, csiDeployment.Spec.Template.Spec.Tolerations) + + // TestCase: Update tolerations in the CSI spec + tolerations = append(tolerations, v1.Toleration{ + Key: "cluster/infra", + Value: "true", + Operator: v1.TolerationOpEqual, + Effect: v1.TaintEffectNoSchedule, + }) + cluster.Spec.CSI.Placement.Tolerations = tolerations + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, tolerations, csiDeployment.Spec.Template.Spec.Tolerations) + + // TestCase: Remove tolerations from the CSI spec + cluster.Spec.CSI.Placement.Tolerations = nil + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Empty(t, csiDeployment.Spec.Template.Spec.Tolerations) +} + +func TestCSIExtDeploymentAnnotationsChange(t *testing.T) { + versionClient := fakek8sclient.NewSimpleClientset() + coreops.SetInstance(coreops.New(versionClient)) + versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ + GitVersion: "v1.29.0", + } + fakeExtClient := fakeextclient.NewSimpleClientset() + apiextensionsops.SetInstance(apiextensionsops.New(fakeExtClient)) + + reregisterComponents() + k8sClient := testutil.FakeK8sClient() + driver := portworx{} + err := driver.Init(k8sClient, runtime.NewScheme(), record.NewFakeRecorder(10)) + require.NoError(t, err) + + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-cluster", + Namespace: "kube-test", + }, + Spec: corev1.StorageClusterSpec{ + Image: "portworx/image:3.2.0", + }, + } + createTelemetrySecret(t, k8sClient, cluster.Namespace) + err = driver.SetDefaultsOnStorageCluster(cluster) + require.NoError(t, err) + + // TestCase: CSI pods and deployments should not have any annotations as it is not defined in the cluster spec. + err = driver.PreInstall(cluster) + require.NoError(t, err) + + csiDeployment := &appsv1.Deployment{} + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Empty(t, csiDeployment.Annotations) + require.Empty(t, csiDeployment.Spec.Template.Annotations) + + // TestCase: Define annotations for the CSI deployment + csiDeploymentAnnotationKey := fmt.Sprintf("%s/%s", k8sutil.Deployment, component.CSIApplicationName) + annotations := map[string]string{ + "key1": "value1", + } + cluster.Spec.Metadata = &corev1.Metadata{ + Annotations: map[string]map[string]string{ + csiDeploymentAnnotationKey: annotations, + }, + } + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, csiDeployment.Annotations, csiDeployment.Spec.Template.Annotations) + require.Len(t, csiDeployment.Annotations, 2) + require.Contains(t, csiDeployment.Annotations, constants.AnnotationCustomAnnotations) + require.Equal(t, "value1", csiDeployment.Annotations["key1"]) + + // TestCase: Add annotations to the CSI deployment + annotations["key2"] = "value2" + cluster.Spec.Metadata.Annotations[csiDeploymentAnnotationKey] = annotations + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, csiDeployment.Annotations, csiDeployment.Spec.Template.Annotations) + require.Len(t, csiDeployment.Annotations, 3) + require.Contains(t, csiDeployment.Annotations, constants.AnnotationCustomAnnotations) + require.Equal(t, "value1", csiDeployment.Annotations["key1"]) + require.Equal(t, "value2", csiDeployment.Annotations["key2"]) + + // TestCase: Change annotation value for the CSI deployment + annotations["key2"] = "value2-changed" + cluster.Spec.Metadata.Annotations[csiDeploymentAnnotationKey] = annotations + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, csiDeployment.Annotations, csiDeployment.Spec.Template.Annotations) + require.Len(t, csiDeployment.Annotations, 3) + require.Contains(t, csiDeployment.Annotations, constants.AnnotationCustomAnnotations) + require.Equal(t, "value1", csiDeployment.Annotations["key1"]) + require.Equal(t, "value2-changed", csiDeployment.Annotations["key2"]) + + // TestCase: Remove an annotation key from the CSI deployment. + // Also ensure that we don't remove any annotations that we did not add. + csiDeployment.Annotations["external-key"] = "external-value" + err = testutil.Update(k8sClient, csiDeployment) + require.NoError(t, err) + + delete(annotations, "key2") + cluster.Spec.Metadata.Annotations[csiDeploymentAnnotationKey] = annotations + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, csiDeployment.Annotations, csiDeployment.Spec.Template.Annotations) + require.Len(t, csiDeployment.Annotations, 3) + require.Contains(t, csiDeployment.Annotations, constants.AnnotationCustomAnnotations) + require.Equal(t, "value1", csiDeployment.Annotations["key1"]) + require.Equal(t, "external-value", csiDeployment.Annotations["external-key"]) + + // TestCase: Remove all annotations from the CSI deployment + cluster.Spec.Metadata.Annotations[csiDeploymentAnnotationKey] = nil + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + err = testutil.Get(k8sClient, csiDeployment, component.CSIApplicationName, cluster.Namespace) + require.NoError(t, err) + require.Equal(t, csiDeployment.Annotations, csiDeployment.Spec.Template.Annotations) + require.Len(t, csiDeployment.Annotations, 1) + require.Equal(t, "external-value", csiDeployment.Annotations["external-key"]) +} + func TestCSIInstallWithCustomKubeletDir(t *testing.T) { mockCtrl := gomock.NewController(t) versionClient := fakek8sclient.NewSimpleClientset() @@ -12979,7 +13626,6 @@ func TestDisableCSI_GTPxVersion2_13(t *testing.T) { podSpec, err = driver.GetStoragePodSpec(cluster, "") require.NoError(t, err) require.Equal(t, 1, len(podSpec.Containers)) - } func TestMonitoringMetricsEnabled(t *testing.T) { diff --git a/drivers/storage/portworx/deployment.go b/drivers/storage/portworx/deployment.go index 5955a53170..bee242344d 100644 --- a/drivers/storage/portworx/deployment.go +++ b/drivers/storage/portworx/deployment.go @@ -544,12 +544,15 @@ func (t *template) kvdbContainer() v1.Container { if t.startPort != pxutil.DefaultStartPort { kvdbTargetPort = t.startPort + 15 } - return v1.Container{ + container := v1.Container{ Name: pxKVDBContainerName, Image: kvdbProxyImage, ImagePullPolicy: t.imagePullPolicy, LivenessProbe: &v1.Probe{ PeriodSeconds: 30, + TimeoutSeconds: int32(1), + SuccessThreshold: int32(1), + FailureThreshold: int32(3), InitialDelaySeconds: 840, ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ @@ -559,7 +562,10 @@ func (t *template) kvdbContainer() v1.Container { }, }, ReadinessProbe: &v1.Probe{ - PeriodSeconds: 10, + PeriodSeconds: 10, + TimeoutSeconds: int32(1), + SuccessThreshold: int32(1), + FailureThreshold: int32(3), ProbeHandler: v1.ProbeHandler{ TCPSocket: &v1.TCPSocketAction{ Port: intstr.FromInt(kvdbTargetPort), @@ -568,6 +574,24 @@ func (t *template) kvdbContainer() v1.Container { }, }, } + + if t.cluster.Spec.Kvdb != nil && + t.cluster.Spec.Kvdb.Resources != nil { + container.Resources = *t.cluster.Spec.Kvdb.Resources + } else { + container.Resources = v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("50Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("200m"), + }, + } + } + + return container } func (t *template) csiRegistrarContainer() *v1.Container { diff --git a/drivers/storage/portworx/deployment_test.go b/drivers/storage/portworx/deployment_test.go index 260639fc4a..fc15250781 100644 --- a/drivers/storage/portworx/deployment_test.go +++ b/drivers/storage/portworx/deployment_test.go @@ -451,6 +451,25 @@ func TestGetKVDBPodSpec(t *testing.T) { actual, err = driver.GetKVDBPodSpec(cluster, nodeName) require.NoError(t, err) assertPodSpecEqual(t, expected, &actual) + + // custom resources + resources := v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("300Mi"), + v1.ResourceCPU: resource.MustParse("300m"), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("500Mi"), + v1.ResourceCPU: resource.MustParse("500m"), + }, + } + cluster.Spec.Kvdb = &corev1.KvdbSpec{ + Resources: &resources, + } + expected.Containers[0].Resources = resources + actual, err = driver.GetKVDBPodSpec(cluster, nodeName) + require.NoError(t, err) + assertPodSpecEqual(t, expected, &actual) } func TestPodSpecWithCustomServiceAccount(t *testing.T) { diff --git a/drivers/storage/portworx/testspec/kvdbPodCustomPort.yaml b/drivers/storage/portworx/testspec/kvdbPodCustomPort.yaml index 21ea51878c..175a07619b 100644 --- a/drivers/storage/portworx/testspec/kvdbPodCustomPort.yaml +++ b/drivers/storage/portworx/testspec/kvdbPodCustomPort.yaml @@ -12,14 +12,27 @@ spec: imagePullPolicy: Always livenessProbe: periodSeconds: 30 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 initialDelaySeconds: 840 tcpSocket: host: 127.0.0.1 port: 10016 readinessProbe: periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 tcpSocket: host: 127.0.0.1 port: 10016 + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi restartPolicy: Always serviceAccountName: portworx diff --git a/drivers/storage/portworx/testspec/kvdbPodDefault.yaml b/drivers/storage/portworx/testspec/kvdbPodDefault.yaml index 317b45d47f..01630e9fb6 100644 --- a/drivers/storage/portworx/testspec/kvdbPodDefault.yaml +++ b/drivers/storage/portworx/testspec/kvdbPodDefault.yaml @@ -12,14 +12,27 @@ spec: imagePullPolicy: Always livenessProbe: periodSeconds: 30 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 initialDelaySeconds: 840 tcpSocket: host: 127.0.0.1 port: 9019 readinessProbe: periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 tcpSocket: host: 127.0.0.1 port: 9019 + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi restartPolicy: Always serviceAccountName: portworx diff --git a/drivers/storage/portworx/testspec/portworxAPIDaemonSet.yaml b/drivers/storage/portworx/testspec/portworxAPIDaemonSet.yaml index f24d50d095..e9edb43290 100644 --- a/drivers/storage/portworx/testspec/portworxAPIDaemonSet.yaml +++ b/drivers/storage/portworx/testspec/portworxAPIDaemonSet.yaml @@ -36,9 +36,19 @@ spec: imagePullPolicy: Always readinessProbe: periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 httpGet: host: 127.0.0.1 path: /status port: 10001 + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi restartPolicy: Always serviceAccountName: portworx diff --git a/drivers/storage/portworx/testspec/portworxAPIDaemonset_2_13.yaml b/drivers/storage/portworx/testspec/portworxAPIDaemonset_2_13.yaml index 4e3fd248a4..93a6126f4a 100644 --- a/drivers/storage/portworx/testspec/portworxAPIDaemonset_2_13.yaml +++ b/drivers/storage/portworx/testspec/portworxAPIDaemonset_2_13.yaml @@ -36,10 +36,20 @@ spec: imagePullPolicy: Always readinessProbe: periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 httpGet: host: 127.0.0.1 path: /status port: 10001 + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi - args: - --v=5 - --csi-address=$(ADDRESS) @@ -50,12 +60,18 @@ spec: - name: KUBE_NODE_NAME valueFrom: fieldRef: - apiVersion: "" + apiVersion: "v1" fieldPath: spec.nodeName image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.3 imagePullPolicy: Always name: csi-node-driver-registrar - resources: {} + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi terminationMessagePath: "" terminationMessagePolicy: "" volumeMounts: diff --git a/pkg/apis/core/v1/storagecluster.go b/pkg/apis/core/v1/storagecluster.go index c401569d3b..baf22985f2 100644 --- a/pkg/apis/core/v1/storagecluster.go +++ b/pkg/apis/core/v1/storagecluster.go @@ -116,6 +116,8 @@ type StorageClusterSpec struct { Resources *v1.ResourceRequirements `json:"resources,omitempty"` // CSI configurations for setting up CSI CSI *CSISpec `json:"csi,omitempty"` + // PortworxAPI is the spec to configure the portworx API component. + PortworxAPI *PortworxAPISpec `json:"portworxApi,omitempty"` // Priority Class Name to be passed to Podspec of px pods for it to be scheduled accordingly PriorityClassName string `json:"priorityClassName,omitempty"` } @@ -158,9 +160,15 @@ type NodeSpec struct { // CSISpec is used to define the CSI configurations type CSISpec struct { - Enabled bool `json:"enabled,omitempty"` - InstallSnapshotController *bool `json:"installSnapshotController,omitempty"` - Topology *CSITopologySpec `json:"topology,omitempty"` + Enabled bool `json:"enabled,omitempty"` + InstallSnapshotController *bool `json:"installSnapshotController,omitempty"` + Topology *CSITopologySpec `json:"topology,omitempty"` + Placement *PlacementSpec `json:"placement,omitempty"` + NodeDriverRegistrar *CSINodeDriverRegistrarSpec `json:"nodeDriverRegistrar,omitempty"` + ExternalProvisioner *CSIExternalProvisionerSpec `json:"externalProvisioner,omitempty"` + Snapshotter *CSISnapshotterSpec `json:"snapshotter,omitempty"` + Resizer *CSIResizerSpec `json:"resizer,omitempty"` + SnapshotController *CSISnapshotControllerSpec `json:"snapshotController,omitempty"` } // CSITopologySpec is used to define the CSI topology configurations @@ -168,6 +176,42 @@ type CSITopologySpec struct { Enabled bool `json:"enabled,omitempty"` } +// CSINodeDriverRegistrarSpec is used to define the CSI node driver registrar configurations +type CSINodeDriverRegistrarSpec struct { + // Resources for csi-node-driver-registrar container, e.g. CPU and memory requests or limits + Resources *v1.ResourceRequirements `json:"resources,omitempty"` +} + +// CSIExternalProvisionerSpec is used to define the CSI external provisioner configurations +type CSIExternalProvisionerSpec struct { + // Resources for csi-external-provisioner container, e.g. CPU and memory requests or limits + Resources *v1.ResourceRequirements `json:"resources,omitempty"` +} + +// CSISnapshotterSpec is used to define the CSI snapshotter configurations +type CSISnapshotterSpec struct { + // Resources for csi-snapshotter container, e.g. CPU and memory requests or limits + Resources *v1.ResourceRequirements `json:"resources,omitempty"` +} + +// CSIResizerSpec is used to define the CSI resizer configurations +type CSIResizerSpec struct { + // Resources for csi-resizer container, e.g. CPU and memory requests or limits + Resources *v1.ResourceRequirements `json:"resources,omitempty"` +} + +// CSISnapshotControllerSpec is used to define the CSI snapshot controller configurations +type CSISnapshotControllerSpec struct { + // Resources for csi snapshot-controller container, e.g. CPU and memory requests or limits + Resources *v1.ResourceRequirements `json:"resources,omitempty"` +} + +// PortworxAPISpec is used to configure the Portworx API component. +type PortworxAPISpec struct { + // Resources for portworx-api container, e.g. CPU and memory requests or limits + Resources *v1.ResourceRequirements `json:"resources,omitempty"` +} + // SecuritySpec is used to define the security configuration for a cluster. type SecuritySpec struct { Enabled bool `json:"enabled,omitempty"` @@ -332,6 +376,8 @@ type KvdbSpec struct { // to authenticate with the kvdb. It could have the username/password // for basic auth, certificate information or ACL token. AuthSecret string `json:"authSecret,omitempty"` + // Resources for portworx-kvdb pods, e.g. CPU and memory requests or limits + Resources *v1.ResourceRequirements `json:"resources,omitempty"` } // NetworkSpec contains network information diff --git a/pkg/apis/core/v1/zz_generated.deepcopy.go b/pkg/apis/core/v1/zz_generated.deepcopy.go index d9dcb973e6..0555398ab1 100644 --- a/pkg/apis/core/v1/zz_generated.deepcopy.go +++ b/pkg/apis/core/v1/zz_generated.deepcopy.go @@ -120,6 +120,111 @@ func (in *AutopilotSpec) DeepCopy() *AutopilotSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSIExternalProvisionerSpec) DeepCopyInto(out *CSIExternalProvisionerSpec) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIExternalProvisionerSpec. +func (in *CSIExternalProvisionerSpec) DeepCopy() *CSIExternalProvisionerSpec { + if in == nil { + return nil + } + out := new(CSIExternalProvisionerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSINodeDriverRegistrarSpec) DeepCopyInto(out *CSINodeDriverRegistrarSpec) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSINodeDriverRegistrarSpec. +func (in *CSINodeDriverRegistrarSpec) DeepCopy() *CSINodeDriverRegistrarSpec { + if in == nil { + return nil + } + out := new(CSINodeDriverRegistrarSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSIResizerSpec) DeepCopyInto(out *CSIResizerSpec) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIResizerSpec. +func (in *CSIResizerSpec) DeepCopy() *CSIResizerSpec { + if in == nil { + return nil + } + out := new(CSIResizerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSISnapshotControllerSpec) DeepCopyInto(out *CSISnapshotControllerSpec) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSISnapshotControllerSpec. +func (in *CSISnapshotControllerSpec) DeepCopy() *CSISnapshotControllerSpec { + if in == nil { + return nil + } + out := new(CSISnapshotControllerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSISnapshotterSpec) DeepCopyInto(out *CSISnapshotterSpec) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSISnapshotterSpec. +func (in *CSISnapshotterSpec) DeepCopy() *CSISnapshotterSpec { + if in == nil { + return nil + } + out := new(CSISnapshotterSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CSISpec) DeepCopyInto(out *CSISpec) { *out = *in @@ -133,6 +238,36 @@ func (in *CSISpec) DeepCopyInto(out *CSISpec) { *out = new(CSITopologySpec) **out = **in } + if in.Placement != nil { + in, out := &in.Placement, &out.Placement + *out = new(PlacementSpec) + (*in).DeepCopyInto(*out) + } + if in.NodeDriverRegistrar != nil { + in, out := &in.NodeDriverRegistrar, &out.NodeDriverRegistrar + *out = new(CSINodeDriverRegistrarSpec) + (*in).DeepCopyInto(*out) + } + if in.ExternalProvisioner != nil { + in, out := &in.ExternalProvisioner, &out.ExternalProvisioner + *out = new(CSIExternalProvisionerSpec) + (*in).DeepCopyInto(*out) + } + if in.Snapshotter != nil { + in, out := &in.Snapshotter, &out.Snapshotter + *out = new(CSISnapshotterSpec) + (*in).DeepCopyInto(*out) + } + if in.Resizer != nil { + in, out := &in.Resizer, &out.Resizer + *out = new(CSIResizerSpec) + (*in).DeepCopyInto(*out) + } + if in.SnapshotController != nil { + in, out := &in.SnapshotController, &out.SnapshotController + *out = new(CSISnapshotControllerSpec) + (*in).DeepCopyInto(*out) + } return } @@ -438,6 +573,11 @@ func (in *KvdbSpec) DeepCopyInto(out *KvdbSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } return } @@ -742,6 +882,27 @@ func (in *PlacementSpec) DeepCopy() *PlacementSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortworxAPISpec) DeepCopyInto(out *PortworxAPISpec) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortworxAPISpec. +func (in *PortworxAPISpec) DeepCopy() *PortworxAPISpec { + if in == nil { + return nil + } + out := new(PortworxAPISpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) { *out = *in @@ -1070,6 +1231,11 @@ func (in *StorageClusterSpec) DeepCopyInto(out *StorageClusterSpec) { *out = new(CSISpec) (*in).DeepCopyInto(*out) } + if in.PortworxAPI != nil { + in, out := &in.PortworxAPI, &out.PortworxAPI + *out = new(PortworxAPISpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/constants/metadata.go b/pkg/constants/metadata.go index d4871759f7..1e484ef38a 100644 --- a/pkg/constants/metadata.go +++ b/pkg/constants/metadata.go @@ -19,6 +19,8 @@ const ( AnnotationEvictVMsDuringUpdate = OperatorPrefix + "/evict-vms-during-update" // AnnotationNodeLabels is the storage pod annotation that contains node labels AnnotationNodeLabels = OperatorPrefix + "/node-labels" + // AnnotationCustomAnnotations is an annotation that contains all the custom annotations a user adds on an object. + AnnotationCustomAnnotations = OperatorPrefix + "/custom-annotations" // AnnotationDisableStorage annotation to disable the storage pods from running (default: false) AnnotationDisableStorage = OperatorPrefix + "/disable-storage" // AnnotationReconcileObject annotation to toggle reconciliation of operator created objects diff --git a/pkg/controller/storagenode/storagenode.go b/pkg/controller/storagenode/storagenode.go index 9eb29109d5..d5f27f1442 100644 --- a/pkg/controller/storagenode/storagenode.go +++ b/pkg/controller/storagenode/storagenode.go @@ -3,17 +3,20 @@ package storagenode import ( "context" "fmt" + "reflect" + "strings" + "time" + "github.com/libopenstorage/openstorage/api" pxutil "github.com/libopenstorage/operator/drivers/storage/portworx/util" "google.golang.org/grpc" - "strings" - "time" "github.com/hashicorp/go-version" apiextensionsops "github.com/portworx/sched-ops/k8s/apiextensions" coreops "github.com/portworx/sched-ops/k8s/core" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -297,21 +300,45 @@ func (c *Controller) syncKVDB( isRecentlyCreatedAfterNodeCordoned = k8s.IsPodRecentlyCreatedAfterNodeCordoned(node, c.nodeInfoMap, cluster) } - if !isBeingDeleted && !isRecentlyCreatedAfterNodeCordoned && len(kvdbPods) == 0 { + if !isBeingDeleted && !isRecentlyCreatedAfterNodeCordoned { pod, err := c.createKVDBPod(cluster, storageNode) if err != nil { return err } - c.log(storageNode).Infof("creating portworx-kvdb pod") - _, err = coreops.Instance().CreatePod(pod) - if err != nil { - return err + if len(kvdbPods) == 0 { + c.log(storageNode).Infof("Creating %s/portworx-kvdb Pod", storageNode.Namespace) + _, err = coreops.Instance().CreatePod(pod) + if err != nil { + return err + } + } else { + // Ideally there will be only one kvdb pod per node, but in case of multiple pods, + // we will delete all of them and create a new one during the next reconciliation. + for _, kvdbPod := range kvdbPods { + if len(pod.Spec.Containers) != len(kvdbPod.Spec.Containers) || + !equality.Semantic.DeepDerivative(pod.Spec.Containers, kvdbPod.Spec.Containers) { + c.log(storageNode).Infof("Deleting %s/%s Pod", kvdbPod.Namespace, kvdbPod.Name) + err = coreops.Instance().DeletePod(kvdbPod.Name, kvdbPod.Namespace, false) + if err != nil { + c.log(storageNode).Warnf("failed to delete pod: %s/%s due to: %v", + kvdbPod.Namespace, kvdbPod.Name, err) + } + } else if !reflect.DeepEqual(pod.Annotations, kvdbPod.Annotations) { + c.log(storageNode).Infof("Updating %s/%s Pod annotations", kvdbPod.Namespace, kvdbPod.Name) + kvdbPod.Annotations = pod.Annotations + _, err = coreops.Instance().UpdatePod(&kvdbPod) + if err != nil { + c.log(storageNode).Warnf("failed to update pod: %s/%s due to: %v", + kvdbPod.Namespace, kvdbPod.Name, err) + } + } + } } } } else { // delete pods if present for _, p := range kvdbPods { - c.log(storageNode).Debugf("deleting kvdb pod: %s/%s", p.Namespace, p.Name) + c.log(storageNode).Debugf("Deleting %s/%s Pod", p.Namespace, p.Name) err = coreops.Instance().DeletePod(p.Name, p.Namespace, false) if err != nil { c.log(storageNode).Warnf("failed to delete pod: %s/%s due to: %v", p.Namespace, p.Name, err) @@ -415,6 +442,7 @@ func (c *Controller) createKVDBPod( GenerateName: fmt.Sprintf("%s-kvdb-", c.Driver.String()), Namespace: storageNode.Namespace, Labels: c.kvdbPodLabels(cluster), + Annotations: util.GetCustomAnnotations(cluster, k8s.Pod, fmt.Sprintf("%s-kvdb", c.Driver.String())), OwnerReferences: []metav1.OwnerReference{*ownerRef}, }, Spec: podSpec, diff --git a/pkg/controller/storagenode/storagenode_test.go b/pkg/controller/storagenode/storagenode_test.go index 6e13294a26..820c902286 100644 --- a/pkg/controller/storagenode/storagenode_test.go +++ b/pkg/controller/storagenode/storagenode_test.go @@ -2,17 +2,15 @@ package storagenode import ( "context" - osdapi "github.com/libopenstorage/openstorage/api" - pxutil "github.com/libopenstorage/operator/drivers/storage/portworx/util" "reflect" "strconv" "testing" "time" "github.com/golang/mock/gomock" + osdapi "github.com/libopenstorage/openstorage/api" apiextensionsops "github.com/portworx/sched-ops/k8s/apiextensions" coreops "github.com/portworx/sched-ops/k8s/core" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -31,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + pxutil "github.com/libopenstorage/operator/drivers/storage/portworx/util" corev1 "github.com/libopenstorage/operator/pkg/apis/core/v1" "github.com/libopenstorage/operator/pkg/client/clientset/versioned/scheme" "github.com/libopenstorage/operator/pkg/constants" @@ -270,7 +269,6 @@ func TestRegisterDeprecatedCRD(t *testing.T) { } func TestReconcile(t *testing.T) { - logrus.SetLevel(logrus.TraceLevel) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() defaultQuantity, _ := resource.ParseQuantity("0") @@ -849,9 +847,8 @@ func TestReconcileWithStorageLabel(t *testing.T) { // TestReconcileKVDB focuses on reconciling a StorageNode which is running KVDB func TestReconcileKVDB(t *testing.T) { - logrus.SetLevel(logrus.TraceLevel) mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() + testNS := "test-ns" clusterName := "test-cluster" cluster := &corev1.StorageCluster{ @@ -983,8 +980,210 @@ func TestReconcileKVDB(t *testing.T) { require.Error(t, err) } -func TestReconcileKVDBOddSatusPodCleanup(t *testing.T) { - logrus.SetLevel(logrus.TraceLevel) +func TestKVDBPodChanges(t *testing.T) { + mockCtrl := gomock.NewController(t) + + testNS := "test-ns" + clusterName := "test-cluster" + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: testNS, + }, + Spec: corev1.StorageClusterSpec{ + Kvdb: &corev1.KvdbSpec{ + Internal: true, + }, + }, + } + + controllerKind := corev1.SchemeGroupVersion.WithKind("StorageCluster") + clusterRef := metav1.NewControllerRef(cluster, controllerKind) + // node1 will not have any kvdb pods. So test will create. + testKVDBNode := "node1" + trueValue := true + kvdbNode := &corev1.StorageNode{ + ObjectMeta: metav1.ObjectMeta{ + Name: testKVDBNode, + Namespace: cluster.Namespace, + OwnerReferences: []metav1.OwnerReference{*clusterRef}, + }, + Status: corev1.NodeStatus{ + Phase: string(corev1.NodeInitStatus), + NodeAttributes: &corev1.NodeAttributes{ + KVDB: &trueValue, + }, + }, + } + + k8sNode := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: testKVDBNode, + }, + Status: v1.NodeStatus{ + Phase: v1.NodeRunning, + }, + } + + coreops.SetInstance(coreops.New(fakek8sclient.NewSimpleClientset())) + k8sClient := testutil.FakeK8sClient(kvdbNode, k8sNode, cluster) + recorder := record.NewFakeRecorder(10) + driver := testutil.MockDriver(mockCtrl) + driver.EXPECT().GetSelectorLabels().Return(nil).AnyTimes() + driver.EXPECT().String().Return("portworx").AnyTimes() + kvdbPodSpec := v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "px-kvdb", + }, + }, + } + kvdbPodSpec.NodeName = testKVDBNode + driver.EXPECT().GetKVDBPodSpec(gomock.Any(), testKVDBNode).Return(kvdbPodSpec, nil) + + controller := Controller{ + client: k8sClient, + recorder: recorder, + Driver: driver, + nodeInfoMap: maps.MakeSyncMap[string, *k8s.NodeInfo](), + } + + // TestCase: KVDB pod does not have any resources as the pod spec does not return any. + request := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: testKVDBNode, + Namespace: testNS, + }, + } + + _, err := controller.Reconcile(context.TODO(), request) + require.NoError(t, err) + + kvdbPods, err := coreops.Instance().GetPods(cluster.Namespace, controller.kvdbPodLabels(cluster)) + require.NoError(t, err) + require.Len(t, kvdbPods.Items, 1) + require.Empty(t, kvdbPods.Items[0].Spec.Containers[0].Resources) + + // TestCase: KVDB pod has resources as given in the pod spec. + kvdbResources := v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + }, + } + kvdbPodSpec.Containers[0].Resources = kvdbResources + driver.EXPECT().GetKVDBPodSpec(gomock.Any(), testKVDBNode).Return(kvdbPodSpec, nil).Times(2) + + _, err = controller.Reconcile(context.TODO(), request) + require.NoError(t, err) + + kvdbPods, err = coreops.Instance().GetPods(cluster.Namespace, controller.kvdbPodLabels(cluster)) + require.NoError(t, err) + // The pod gets deleted first and then recreated with the new resources. + require.Len(t, kvdbPods.Items, 0) + + _, err = controller.Reconcile(context.TODO(), request) + require.NoError(t, err) + + kvdbPods, err = coreops.Instance().GetPods(cluster.Namespace, controller.kvdbPodLabels(cluster)) + require.NoError(t, err) + require.Len(t, kvdbPods.Items, 1) + require.Equal(t, kvdbResources, kvdbPods.Items[0].Spec.Containers[0].Resources) + + // TestCase: KVDB pod should not have any annotations by default + require.Empty(t, kvdbPods.Items[0].Annotations) + + // TestCase: KVDB pod should have annotations if provided in the pod spec + driver.EXPECT().GetKVDBPodSpec(gomock.Any(), testKVDBNode).Return(kvdbPodSpec, nil).AnyTimes() + kvdbAnnotations := map[string]string{ + "key1": "value1", + } + cluster.Spec.Metadata = &corev1.Metadata{ + Annotations: map[string]map[string]string{ + "pod/portworx-kvdb": kvdbAnnotations, + }, + } + err = testutil.Update(k8sClient, cluster) + require.NoError(t, err) + + _, err = controller.Reconcile(context.TODO(), request) + require.NoError(t, err) + + kvdbPods, err = coreops.Instance().GetPods(cluster.Namespace, controller.kvdbPodLabels(cluster)) + require.NoError(t, err) + require.Len(t, kvdbPods.Items, 1) + require.Equal(t, kvdbAnnotations, kvdbPods.Items[0].Annotations) + + // TestCase: KVDB pod should have new annotations if added in the spec. + kvdbAnnotations["key2"] = "value2" + cluster.Spec.Metadata = &corev1.Metadata{ + Annotations: map[string]map[string]string{ + "pod/portworx-kvdb": kvdbAnnotations, + }, + } + err = testutil.Update(k8sClient, cluster) + require.NoError(t, err) + + _, err = controller.Reconcile(context.TODO(), request) + require.NoError(t, err) + + kvdbPods, err = coreops.Instance().GetPods(cluster.Namespace, controller.kvdbPodLabels(cluster)) + require.NoError(t, err) + require.Len(t, kvdbPods.Items, 1) + require.Equal(t, kvdbAnnotations, kvdbPods.Items[0].Annotations) + + // TestCase: KVDB pod annotations should have updated values if changed in the spec. + kvdbAnnotations["key2"] = "value2-changed" + cluster.Spec.Metadata = &corev1.Metadata{ + Annotations: map[string]map[string]string{ + "pod/portworx-kvdb": kvdbAnnotations, + }, + } + err = testutil.Update(k8sClient, cluster) + require.NoError(t, err) + + _, err = controller.Reconcile(context.TODO(), request) + require.NoError(t, err) + + kvdbPods, err = coreops.Instance().GetPods(cluster.Namespace, controller.kvdbPodLabels(cluster)) + require.NoError(t, err) + require.Len(t, kvdbPods.Items, 1) + require.Equal(t, kvdbAnnotations, kvdbPods.Items[0].Annotations) + + // TestCase: KVDB pod annotations should remove key if removed from the spec. + delete(kvdbAnnotations, "key2") + cluster.Spec.Metadata = &corev1.Metadata{ + Annotations: map[string]map[string]string{ + "pod/portworx-kvdb": kvdbAnnotations, + }, + } + err = testutil.Update(k8sClient, cluster) + require.NoError(t, err) + + _, err = controller.Reconcile(context.TODO(), request) + require.NoError(t, err) + + kvdbPods, err = coreops.Instance().GetPods(cluster.Namespace, controller.kvdbPodLabels(cluster)) + require.NoError(t, err) + require.Len(t, kvdbPods.Items, 1) + require.Equal(t, kvdbAnnotations, kvdbPods.Items[0].Annotations) + + // TestCase: KVDB pod annotations should all annotations if not specified in the spec. + cluster.Spec.Metadata = &corev1.Metadata{ + Annotations: map[string]map[string]string{}, + } + err = testutil.Update(k8sClient, cluster) + require.NoError(t, err) + + _, err = controller.Reconcile(context.TODO(), request) + require.NoError(t, err) + + kvdbPods, err = coreops.Instance().GetPods(cluster.Namespace, controller.kvdbPodLabels(cluster)) + require.NoError(t, err) + require.Len(t, kvdbPods.Items, 1) + require.Empty(t, kvdbPods.Items[0].Annotations) +} + +func TestReconcileKVDBOddStatusPodCleanup(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() testNS := "test-ns" @@ -1271,7 +1470,6 @@ func TestReconcileKVDBWithNodeChanges(t *testing.T) { } func TestSyncStorageNodeErrors(t *testing.T) { - logrus.SetLevel(logrus.TraceLevel) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() diff --git a/pkg/util/k8s/k8s.go b/pkg/util/k8s/k8s.go index df6b917017..7b7151ea32 100644 --- a/pkg/util/k8s/k8s.go +++ b/pkg/util/k8s/k8s.go @@ -40,8 +40,10 @@ import ( // Constants for k8s object kinds const ( - Pod = "pod" - Service = "service" + Pod = "pod" + Service = "service" + Deployment = "deployment" + DaemonSet = "daemonset" NodeRoleLabelMaster = "node-role.kubernetes.io/master" NodeRoleLabelControlPlane = "node-role.kubernetes.io/control-plane" @@ -1000,7 +1002,7 @@ func CreateOrUpdateDeployment( existingDeployment, ) if errors.IsNotFound(err) { - logrus.Infof("Creating %s Deployment", deployment.Name) + logrus.Infof("Creating %s/%s Deployment", deployment.Namespace, deployment.Name) return k8sClient.Create(context.TODO(), deployment) } else if err != nil { return err @@ -1012,7 +1014,7 @@ func CreateOrUpdateDeployment( } } - logrus.Debugf("Updating %s Deployment", deployment.Name) + logrus.Debugf("Updating %s/%s Deployment", deployment.Namespace, deployment.Name) return k8sClient.Update(context.TODO(), deployment) } @@ -1071,7 +1073,7 @@ func CreateOrUpdateStatefulSet( existingSS, ) if errors.IsNotFound(err) { - logrus.Infof("Creating %s StatefulSet", ss.Name) + logrus.Infof("Creating %s/%s StatefulSet", ss.Namespace, ss.Name) return k8sClient.Create(context.TODO(), ss) } else if err != nil { return err @@ -1083,7 +1085,7 @@ func CreateOrUpdateStatefulSet( } } - logrus.Debugf("Updating %s StatefulSet", ss.Name) + logrus.Debugf("Updating %s/%s StatefulSet", ss.Namespace, ss.Name) return k8sClient.Update(context.TODO(), ss) } @@ -1189,6 +1191,7 @@ func CreateOrUpdateDaemonSet( } existingDS.Labels = ds.Labels + existingDS.Annotations = ds.Annotations existingDS.Spec = ds.Spec logrus.Debugf("Updating %s/%s DaemonSet", ds.Namespace, ds.Name) @@ -1823,7 +1826,7 @@ func CreateOrUpdateConsolePlugin( ) if errors.IsNotFound(err) { - logrus.Infof("Creating %s Consoleplugin", cp.Name) + logrus.Infof("Creating %s/%s ConsolePlugin", cp.Namespace, cp.Name) return k8sClient.Create(context.TODO(), cp) } else if err != nil { return err @@ -1839,7 +1842,7 @@ func CreateOrUpdateConsolePlugin( if modified || len(cp.OwnerReferences) > len(existingPlugin.OwnerReferences) { cp.ResourceVersion = existingPlugin.ResourceVersion - logrus.Infof("Updating Console Plugin %s/%s", cp.Namespace, cp.Name) + logrus.Infof("Updating %s/%s ConsolePlugin", cp.Namespace, cp.Name) return k8sClient.Update(context.TODO(), cp) } return nil diff --git a/pkg/util/util.go b/pkg/util/util.go index 87d458df3e..83240d91f5 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -2,6 +2,7 @@ package util import ( "context" + "encoding/json" "fmt" "net" "path" @@ -338,13 +339,21 @@ func DeepEqualObjects( // DeepEqualPodTemplate compares if two pod template specs are same. func DeepEqualPodTemplate(t1, t2 *v1.PodTemplateSpec) (bool, error) { // DeepDerivative will return true if first argument is nil, hence check the length of volumes. - // The reason we don't2 use deepEqual for volumes is k8s API server may add defaultMode to it. - if !equality.Semantic.DeepDerivative(t1.Spec.Containers, t2.Spec.Containers) { + // The reason we don't use deepEqual for volumes is k8s API server may add defaultMode to it. + if len(t1.Spec.Containers) != len(t2.Spec.Containers) || + !equality.Semantic.DeepDerivative(t1.Spec.Containers, t2.Spec.Containers) { return false, fmt.Errorf("containers not equal, first: %+v, second: %+v", t1.Spec.Containers, t2.Spec.Containers) } - if !(len(t1.Spec.Volumes) == len(t2.Spec.Volumes) && - equality.Semantic.DeepDerivative(t1.Spec.Volumes, t2.Spec.Volumes)) { + for i := range t1.Spec.Containers { + if !equality.Semantic.DeepEqual(t1.Spec.Containers[i].Resources, t2.Spec.Containers[i].Resources) { + return false, fmt.Errorf("resources for container %s not equal, first: %+v, second: %+v", + t1.Spec.Containers[i].Name, t1.Spec.Containers[i].Resources, t2.Spec.Containers[i].Resources) + } + } + + if len(t1.Spec.Volumes) != len(t2.Spec.Volumes) || + !equality.Semantic.DeepDerivative(t1.Spec.Volumes, t2.Spec.Volumes) { return false, fmt.Errorf("volumes not equal, first: %+v, second: %+v", t1.Spec.Volumes, t2.Spec.Volumes) } @@ -453,6 +462,59 @@ func GetCustomAnnotations( return nil } +// GetCombinedAnnotations returns all annotations that are in the existing list and the custom annotations. +// We store the custom annotations in a special annotation so that we can detect when an annotation that +// was added by us is removed. +func GetCombinedAnnotations( + cluster *corev1.StorageCluster, + k8sObjKind string, + componentName string, + existing map[string]string, +) map[string]string { + newCustomAnnotations := GetCustomAnnotations(cluster, k8sObjKind, componentName) + + var oldCustomAnnotations map[string]string + if encodedAnnotations, ok := existing[constants.AnnotationCustomAnnotations]; ok { + if err := json.Unmarshal([]byte(encodedAnnotations), &oldCustomAnnotations); err != nil { + logrus.Errorf("Failed to unmarshal custom annotations: %v", err) + } + } + + output := make(map[string]string) + for k, v := range existing { + if k == constants.AnnotationCustomAnnotations { + continue + } + if newVal, ok := newCustomAnnotations[k]; ok { + // Use the newer value if it exists + output[k] = newVal + } else if _, ok := oldCustomAnnotations[k]; !ok { + // If the annotation does not exist in the new annotations, nor was it present in the previous + // custom annotations, then it is added by an external entity. We should keep that value. + output[k] = v + } + // Remove the annotation if it does not exist in the new annotations and was present in the previous. + } + + // Copy the new annotations and give it's value more precedence in case of a matching key. + for k, v := range newCustomAnnotations { + output[k] = v + } + + // Keep a copy of the current annotations, so we detect when an annotation + // is removed next time by comparing it with this copy. + if len(newCustomAnnotations) > 0 { + encodedAnnotations, err := json.Marshal(newCustomAnnotations) + if err != nil { + logrus.Errorf("Failed to marshal custom annotations: %v", err) + } else { + output[constants.AnnotationCustomAnnotations] = string(encodedAnnotations) + } + } + + return output +} + // GetCustomLabels returns custom labels for different StorageCluster components from spec func GetCustomLabels( cluster *corev1.StorageCluster, From cbc7f9d7cccdc0428650c4fc1a40d71a7fb33fe7 Mon Sep 17 00:00:00 2001 From: Shreyas Sreenivas Date: Mon, 19 Aug 2024 17:54:39 +0530 Subject: [PATCH 2/2] Fixing unit tests --- drivers/storage/portworx/component/csi.go | 82 +++++++++++++------ .../portworx/component/portworx_basic.go | 1 + drivers/storage/portworx/components_test.go | 39 +++++++-- 3 files changed, 93 insertions(+), 29 deletions(-) diff --git a/drivers/storage/portworx/component/csi.go b/drivers/storage/portworx/component/csi.go index 25fb48c55e..b4c55204bc 100644 --- a/drivers/storage/portworx/component/csi.go +++ b/drivers/storage/portworx/component/csi.go @@ -442,24 +442,28 @@ func (c *csi) createDeployment( return err } - if cluster.Spec.CSI.InstallSnapshotController != nil && - *cluster.Spec.CSI.InstallSnapshotController && - cluster.Status.DesiredImages.CSISnapshotController != "" { - // Check if a snapshot controller was installed already using image "snapshot-controller", - // Only do this once to avoid scanning all pods frequently. - if c.csiSnapshotControllerPreInstalled == nil { - if err := c.findPreinstalledCSISnapshotController(); err != nil { - return err - } - } - } + // if cluster.Spec.CSI.InstallSnapshotController != nil && + // *cluster.Spec.CSI.InstallSnapshotController && + // cluster.Status.DesiredImages.CSISnapshotController != "" { + // // Check if a snapshot controller was installed already using image "snapshot-controller", + // // Only do this once to avoid scanning all pods frequently. + // if c.csiSnapshotControllerPreInstalled == nil { + // if err := c.findPreinstalledCSISnapshotController(); err != nil { + // return err + // } + // } + // } updatedTopologySpreadConstraints, err := util.GetTopologySpreadConstraints(c.k8sClient, csiDeploymentTemplateSelectorLabels) if err != nil { return err } - deployment := c.getCSIDeploymentSpec(cluster, csiConfig, updatedTopologySpreadConstraints, existingDeployment.Annotations, ownerRef) + deployment, err := c.getCSIDeploymentSpec(cluster, csiConfig, updatedTopologySpreadConstraints, existingDeployment.Annotations, ownerRef) + if err != nil { + return err + } + isPodTemplateEqual, _ := util.DeepEqualPodTemplate(&deployment.Spec.Template, &existingDeployment.Spec.Template) modified := !isPodTemplateEqual || @@ -529,7 +533,7 @@ func (c *csi) getCSIDeploymentSpec( topologySpreadConstraints []v1.TopologySpreadConstraint, existingAnnotations map[string]string, ownerRef *metav1.OwnerReference, -) *appsv1.Deployment { +) (*appsv1.Deployment, error) { replicas := int32(3) leaderElectionType := "leases" provisionerLeaderElectionType := "leases" @@ -552,34 +556,62 @@ func (c *csi) getCSIDeploymentSpec( cluster, cluster.Status.DesiredImages.CSIProvisioner, ) + + if provisionerImage == "" { + return nil, fmt.Errorf("csi provisioner image not found") + } + if csiConfig.IncludeAttacher && cluster.Status.DesiredImages.CSIAttacher != "" { attacherImage = util.GetImageURN( cluster, cluster.Status.DesiredImages.CSIAttacher, ) + + if attacherImage == "" { + return nil, fmt.Errorf("csi attacher image not found") + } } - if csiConfig.IncludeSnapshotter && cluster.Status.DesiredImages.CSISnapshotter != "" { + if csiConfig.IncludeSnapshotter { snapshotterImage = util.GetImageURN( cluster, cluster.Status.DesiredImages.CSISnapshotter, ) + + if snapshotterImage == "" { + return nil, fmt.Errorf("csi snapshotter image not found") + } } - if csiConfig.IncludeResizer && cluster.Status.DesiredImages.CSIResizer != "" { + if csiConfig.IncludeResizer { resizerImage = util.GetImageURN( cluster, cluster.Status.DesiredImages.CSIResizer, ) + + if resizerImage == "" { + return nil, fmt.Errorf("csi resizer image not found") + } } if cluster.Spec.CSI.InstallSnapshotController != nil && *cluster.Spec.CSI.InstallSnapshotController && - cluster.Status.DesiredImages.CSISnapshotController != "" && - !*c.csiSnapshotControllerPreInstalled { - snapshotControllerImage = util.GetImageURN( - cluster, - cluster.Status.DesiredImages.CSISnapshotController, - ) - + cluster.Status.DesiredImages.CSISnapshotController != "" { + // Check if a snapshot controller was installed already using image "snapshot-controller", + // Only do this once to avoid scanning all pods frequently. + if c.csiSnapshotControllerPreInstalled == nil { + if err := c.findPreinstalledCSISnapshotController(); err != nil { + return nil, err + } + } + if !*c.csiSnapshotControllerPreInstalled { + snapshotControllerImage = util.GetImageURN( + cluster, + cluster.Status.DesiredImages.CSISnapshotController, + ) + } + } else if cluster.Spec.CSI.InstallSnapshotController != nil && + *cluster.Spec.CSI.InstallSnapshotController && + cluster.Status.DesiredImages.CSISnapshotController == "" { + return nil, fmt.Errorf("csi snapshot controller image not found") } if csiConfig.IncludeHealthMonitorController && cluster.Status.DesiredImages.CSIHealthMonitorController != "" { @@ -587,6 +619,10 @@ func (c *csi) getCSIDeploymentSpec( cluster, cluster.Status.DesiredImages.CSIHealthMonitorController, ) + + if healthMonitorControllerImage == "" { + return nil, fmt.Errorf("csi health monitor controller image not found") + } } var args []string @@ -885,7 +921,7 @@ func (c *csi) getCSIDeploymentSpec( } deployment.Spec.Template.ObjectMeta = k8sutil.AddManagedByOperatorLabel(deployment.Spec.Template.ObjectMeta) - return deployment + return deployment, nil } func (c *csi) createStatefulSet( diff --git a/drivers/storage/portworx/component/portworx_basic.go b/drivers/storage/portworx/component/portworx_basic.go index e7eebcc430..c080610093 100644 --- a/drivers/storage/portworx/component/portworx_basic.go +++ b/drivers/storage/portworx/component/portworx_basic.go @@ -592,6 +592,7 @@ func refreshTokenIfNeeded(secret *v1.Secret, cluster *corev1.StorageCluster) (bo if err != nil { return false, err } + if needRefreshToken { if err := refreshToken(secret, cluster); err != nil { return false, fmt.Errorf("failed to refresh the token secret for px container: %w", err) diff --git a/drivers/storage/portworx/components_test.go b/drivers/storage/portworx/components_test.go index 4c97c71cd7..8489028b6d 100644 --- a/drivers/storage/portworx/components_test.go +++ b/drivers/storage/portworx/components_test.go @@ -667,7 +667,14 @@ func TestPortworxAPIDaemonSetAlwaysDeploys(t *testing.T) { } func TestPortworxAPIResourcesChange(t *testing.T) { - coreops.SetInstance(coreops.New(fakek8sclient.NewSimpleClientset())) + mockCtrl := gomock.NewController(t) + versionClient := fakek8sclient.NewSimpleClientset() + versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ + GitVersion: "v1.29.0", + } + + setUpMockCoreOps(mockCtrl, versionClient) + reregisterComponents() k8sClient := testutil.FakeK8sClient() driver := portworx{} @@ -779,7 +786,14 @@ func TestPortworxAPIResourcesChange(t *testing.T) { } func TestPortworxAPIAnnotationsChange(t *testing.T) { - coreops.SetInstance(coreops.New(fakek8sclient.NewSimpleClientset())) + mockCtrl := gomock.NewController(t) + versionClient := fakek8sclient.NewSimpleClientset() + versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ + GitVersion: "v1.29.0", + } + + setUpMockCoreOps(mockCtrl, versionClient) + reregisterComponents() k8sClient := testutil.FakeK8sClient() driver := portworx{} @@ -8280,11 +8294,14 @@ func TestCSI_0_3_NodeAffinityChange(t *testing.T) { } func TestCSIExtResourcesChange(t *testing.T) { + mockCtrl := gomock.NewController(t) versionClient := fakek8sclient.NewSimpleClientset() - coreops.SetInstance(coreops.New(versionClient)) versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ GitVersion: "v1.29.0", } + + setUpMockCoreOps(mockCtrl, versionClient) + fakeExtClient := fakeextclient.NewSimpleClientset() apiextensionsops.SetInstance(apiextensionsops.New(fakeExtClient)) @@ -8400,11 +8417,14 @@ func TestCSIExtResourcesChange(t *testing.T) { } func TestCSIExtSpecificNodeAffinityChange(t *testing.T) { + mockCtrl := gomock.NewController(t) versionClient := fakek8sclient.NewSimpleClientset() - coreops.SetInstance(coreops.New(versionClient)) versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ GitVersion: "v1.29.0", } + + setUpMockCoreOps(mockCtrl, versionClient) + fakeExtClient := fakeextclient.NewSimpleClientset() apiextensionsops.SetInstance(apiextensionsops.New(fakeExtClient)) @@ -8492,11 +8512,14 @@ func TestCSIExtSpecificNodeAffinityChange(t *testing.T) { } func TestCSIExtSpecificTolerationsChange(t *testing.T) { + mockCtrl := gomock.NewController(t) versionClient := fakek8sclient.NewSimpleClientset() - coreops.SetInstance(coreops.New(versionClient)) versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ GitVersion: "v1.29.0", } + + setUpMockCoreOps(mockCtrl, versionClient) + fakeExtClient := fakeextclient.NewSimpleClientset() apiextensionsops.SetInstance(apiextensionsops.New(fakeExtClient)) @@ -8579,11 +8602,14 @@ func TestCSIExtSpecificTolerationsChange(t *testing.T) { } func TestCSIExtDeploymentAnnotationsChange(t *testing.T) { + mockCtrl := gomock.NewController(t) versionClient := fakek8sclient.NewSimpleClientset() - coreops.SetInstance(coreops.New(versionClient)) versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ GitVersion: "v1.29.0", } + + setUpMockCoreOps(mockCtrl, versionClient) + fakeExtClient := fakeextclient.NewSimpleClientset() apiextensionsops.SetInstance(apiextensionsops.New(fakeExtClient)) @@ -16503,6 +16529,7 @@ func TestTelemetryCCMProxy(t *testing.T) { func TestTelemetryCCMGoEnableAndDisable(t *testing.T) { mockCtrl := gomock.NewController(t) setUpMockCoreOps(mockCtrl, fakek8sclient.NewSimpleClientset()) + reregisterComponents() k8sClient := testutil.FakeK8sClient() driver := portworx{}