diff --git a/kubernetes/apps/rook-ceph/kustomization.yaml b/kubernetes/apps/rook-ceph/kustomization.yaml new file mode 100644 index 00000000..dcaf9188 --- /dev/null +++ b/kubernetes/apps/rook-ceph/kustomization.yaml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # Pre Flux-Kustomizations + - ./namespace.yaml + # Flux-Kustomizations + - ./rook-ceph/ks.yaml diff --git a/kubernetes/apps/rook-ceph/namespace.yaml b/kubernetes/apps/rook-ceph/namespace.yaml new file mode 100644 index 00000000..4f4d74a8 --- /dev/null +++ b/kubernetes/apps/rook-ceph/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: rook-ceph + labels: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml b/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml new file mode 100644 index 00000000..202a4e68 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml @@ -0,0 +1,36 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/helm.toolkit.fluxcd.io/helmrelease_v2beta1.json +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: rook-ceph-operator + namespace: rook-ceph +spec: + interval: 30m + timeout: 15m + chart: + spec: + chart: rook-ceph + version: v1.12.2 + sourceRef: + kind: HelmRepository + name: rook-ceph + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + csi: + csiAddons: + enabled: true + # Set when encryption/compression is enabled + cephFSKernelMountOptions: ms_mode=secure + monitoring: + enabled: true diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml b/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml new file mode 100644 index 00000000..2e85a0a6 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: rook-ceph +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/gatus.yaml b/kubernetes/apps/rook-ceph/rook-ceph/cluster/gatus.yaml new file mode 100644 index 00000000..f7882285 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/gatus.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rook-gatus-ep + namespace: rook-ceph + labels: + gatus.io/enabled: "true" +data: + config.yaml: | + endpoints: + - name: rook + group: guarded + url: 1.1.1.1 + interval: 1m + ui: + hide-hostname: true + hide-url: true + dns: + query-name: rook.k8s.dc2.local + query-type: A + conditions: + - "len([BODY]) == 0" + alerts: + - type: pushover diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml new file mode 100644 index 00000000..e6381142 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -0,0 +1,183 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/helm.toolkit.fluxcd.io/helmrelease_v2beta1.json +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: rook-ceph-cluster + namespace: rook-ceph +spec: + interval: 30m + timeout: 15m + chart: + spec: + chart: rook-ceph-cluster + version: v1.12.2 + sourceRef: + kind: HelmRepository + name: rook-ceph + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + # TODO: Add monitoring and ingress + # monitoring: + # enabled: true + # createPrometheusRules: true + # ingress: + # dashboard: + # ingressClassName: internal + # host: + # name: &host rook.k8s.dc2.local + # path: / + # tls: + # - hosts: + # - *host + # toolbox: + # enabled: true + configOverride: | + [global] + bdev_enable_discard = true + bdev_async_discard = true + osd_class_update_on_start = false + cephClusterSpec: + network: + provider: host + connections: + compression: + enabled: true + requireMsgr2: true + crashCollector: + disable: false + dashboard: + enabled: true + urlPrefix: / + ssl: false + storage: + useAllNodes: false + useAllDevices: false + config: + osdsPerDevice: "1" + nodes: + - name: k8s1 + devices: + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:1/8:0:1:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:2/8:0:2:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:3/8:0:3:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:4/8:0:4:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:5/8:0:5:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:6/8:0:6:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:7/8:0:7:0 + - name: k8s2 + devices: + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host7/target7:0:1/7:0:1:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host7/target7:0:2/7:0:2:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host7/target7:0:3/7:0:3:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host7/target7:0:4/7:0:4:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host7/target7:0:5/7:0:5:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host7/target7:0:6/7:0:6:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host7/target7:0:7/7:0:7:0 + - name: k8s3 + devices: + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:1/8:0:1:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:2/8:0:2:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:3/8:0:3:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:4/8:0:4:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:5/8:0:5:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:6/8:0:6:0 + - name: /pci0000:00/0000:00:02.2/0000:02:00.0/host8/target8:0:7/8:0:7:0 + cephBlockPools: + - name: ceph-blockpool + spec: + failureDomain: host + replicated: + size: 6 + storageClass: + enabled: true + name: ceph-block + isDefault: true + reclaimPolicy: Delete + allowVolumeExpansion: true + parameters: + imageFormat: "2" + imageFeatures: layering + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + csi.storage.k8s.io/fstype: ext4 + cephFileSystems: + - name: ceph-filesystem + spec: + metadataPool: + replicated: + size: 6 + dataPools: + - failureDomain: host + replicated: + size: 6 + name: data0 + metadataServer: + activeCount: 1 + activeStandby: true + resources: + requests: + cpu: 1000m + memory: 4Gi + limits: + memory: 4Gi + storageClass: + enabled: true + isDefault: false + name: ceph-filesystem + pool: data0 + reclaimPolicy: Delete + allowVolumeExpansion: true + parameters: + csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + csi.storage.k8s.io/fstype: ext4 + cephObjectStores: + - name: ceph-objectstore + spec: + metadataPool: + failureDomain: host + replicated: + size: 6 + dataPool: + failureDomain: host + erasureCoded: + dataChunks: 4 + codingChunks: 2 + preservePoolsOnDelete: true + gateway: + port: 80 + resources: + requests: + cpu: 1000m + memory: 1Gi + limits: + memory: 2Gi + instances: 2 + healthCheck: + bucket: + interval: 60s + storageClass: + enabled: true + name: ceph-bucket + reclaimPolicy: Delete + parameters: + region: us-east-1 diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml b/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml new file mode 100644 index 00000000..0830f337 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: rook-ceph +resources: + - ./helmrelease.yaml + #TODO: Re-enable after promethus and ingress + # - ./gatus.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml b/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml new file mode 100644 index 00000000..7fe4d79f --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml @@ -0,0 +1,36 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-apps-rook-ceph + namespace: flux-system +spec: + path: ./kubernetes/apps/rook-ceph/rook-ceph/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m +--- +# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-apps-rook-ceph-cluster + namespace: flux-system +spec: + dependsOn: + - name: cluster-apps-rook-ceph + path: ./kubernetes/apps/rook-ceph/rook-ceph/cluster + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 15m