diff --git a/CHANGELOG.md b/CHANGELOG.md index 7644d4fc..a23907c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Chart: Support multiple service account issuers.\ + This is used for example in the migration from Vintage AWS clusters to CAPA. Multiple issuers were previously supported only through internal chart values (this change removes `internal.migration.irsaAdditionalDomain`). The internal annotation `aws.giantswarm.io/irsa-additional-domain` on AWSMachineTemplate objects is changed to plural `aws.giantswarm.io/irsa-trust-domains` on the AWSCluster object. + +### Changed + +- Chart: Update `cluster` to v1.0.1. + ## [1.3.0] - 2024-07-25 ### Changed @@ -140,7 +149,6 @@ Cluster upgrade steps are the following: - Worker nodes - Add `nonRootVolumes` fields to mount `/var/lib` and `/var/log` on separate disk volumes. - BREAKING CHANGE: values `global.controlplane.containerdVolumeSizeGB` and `global.controlplane.kubeletVolumeSizeGB` merged into single value `.global.controlPlane.libVolumeSizeGB` which define size of disk volume used for `/var/lib` mount point. - ### Changed - Control-plane nodes - combine kubelet disk `/var/lib/kubelet` and containerd disk `/var/lib/containerd` into single disk `/var/lib` to share the volume space and save cost. diff --git a/helm/cluster-aws/Chart.lock b/helm/cluster-aws/Chart.lock index 14502787..65a518a2 100644 --- a/helm/cluster-aws/Chart.lock +++ b/helm/cluster-aws/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: cluster repository: https://giantswarm.github.io/cluster-catalog - version: 1.0.0 + version: 1.0.1 - name: cluster-shared repository: https://giantswarm.github.io/cluster-catalog version: 0.7.1 -digest: sha256:68d3cac7c3d274582555c4961171fd053f47e3e562885f3d992f64353a1f9398 -generated: "2024-07-24T14:57:58.852388376+02:00" +digest: sha256:62d157b208d68e6f61bfa0e73088dbcb241a09f6b1f8846c2a6b6f81b1e6a275 +generated: "2024-09-19T16:23:41.237563+02:00" diff --git a/helm/cluster-aws/Chart.yaml b/helm/cluster-aws/Chart.yaml index be524c22..599f3963 100644 --- a/helm/cluster-aws/Chart.yaml +++ b/helm/cluster-aws/Chart.yaml @@ -16,7 +16,7 @@ restrictions: - capa dependencies: - name: cluster - version: "1.0.0" + version: "1.0.1" repository: https://giantswarm.github.io/cluster-catalog - name: cluster-shared version: "0.7.1" diff --git a/helm/cluster-aws/README.md b/helm/cluster-aws/README.md index 02bff39f..078ad3fe 100644 --- a/helm/cluster-aws/README.md +++ b/helm/cluster-aws/README.md @@ -352,8 +352,6 @@ For Giant Swarm internal use only, not stable, or not supported by UIs. | **Property** | **Description** | **More Details** | | :----------- | :-------------- | :--------------- | | `internal.hashSalt` | **Hash salt** - If specified, this token is used as a salt to the hash suffix of some resource names. Can be used to force-recreate some resources.|**Type:** `string`
| -| `internal.migration` | **Migration values** - Section used for migration of cluster from vintage to CAPI|**Type:** `object`
| -| `internal.migration.irsaAdditionalDomain` | **IRSA additional domain** - Additional domain to be added to IRSA trust relationship.|**Type:** `string`
| ### Kubectl image Properties within the `.kubectlImage` top-level object @@ -437,7 +435,7 @@ Information about the workload cluster release. | **Property** | **Description** | **More Details** | | :----------- | :-------------- | :--------------- | | `baseDomain` | **Base DNS domain**|**Type:** `string`
| -| `cluster` | **Cluster** - Helm values for the provider-independent cluster chart|**Type:** `object`
**Default:** `{"providerIntegration":{"apps":{"capiNodeLabeler":{"enable":true},"certExporter":{"enable":true},"certManager":{"enable":true},"chartOperatorExtensions":{"enable":true},"cilium":{"configTemplateName":"awsCiliumHelmValues","enable":true},"ciliumServiceMonitors":{"enable":true},"clusterAutoscaler":{"configTemplateName":"awsClusterAutoscalerHelmValues","enable":true},"coreDns":{"configTemplateName":"awsCorednsHelmValues","enable":true},"etcdKubernetesResourcesCountExporter":{"enable":true},"externalDns":{"configTemplateName":"awsExternalDnsHelmValues","enable":true},"k8sAuditMetrics":{"enable":true},"k8sDnsNodeCache":{"enable":true},"metricsServer":{"enable":true},"netExporter":{"enable":true},"networkPolicies":{"configTemplateName":"awsNetworkPoliciesHelmValues","enable":true},"nodeExporter":{"enable":true},"observabilityBundle":{"enable":true},"prometheusBlackboxExporter":{"enable":true},"securityBundle":{"configTemplateName":"awsSecurityBundleHelmValues","enable":true},"teleportKubeAgent":{"enable":true},"verticalPodAutoscaler":{"enable":true},"verticalPodAutoscalerCrd":{"enable":true}},"clusterAnnotationsTemplateName":"awsConnectivityLabels","components":{"systemd":{"timesyncd":{"ntp":["169.254.169.123"]}}},"connectivity":{"proxy":{"noProxy":{"templateName":"awsNoProxyList","value":["elb.amazonaws.com","169.254.169.254"]}}},"controlPlane":{"kubeadmConfig":{"clusterConfiguration":{"apiServer":{"apiAudiences":{"templateName":"awsApiServerApiAudiences"},"serviceAccountIssuer":{"templateName":"awsIrsaServiceAccountIssuer"}}},"files":[],"ignition":{"containerLinuxConfig":{"additionalConfig":{"storage":{"filesystems":[{"mount":{"device":"/dev/xvdc","format":"xfs","label":"etcd","wipeFilesystem":true},"name":"etcd"},{"mount":{"device":"/dev/xvdd","format":"xfs","label":"lib","wipeFilesystem":true},"name":"lib"},{"mount":{"device":"/dev/xvde","format":"xfs","label":"log","wipeFilesystem":true},"name":"log"}]},"systemd":{"units":[{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/etcd","where":"/var/lib/etcd"},"unit":{"defaultDependencies":false,"description":"etcd volume"}},"enabled":true,"name":"var-lib-etcd.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/lib","where":"/var/lib"},"unit":{"defaultDependencies":false,"description":"var lib volume"}},"enabled":true,"name":"var-lib.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/log","where":"/var/log"},"unit":{"defaultDependencies":false,"description":"log volume"}},"enabled":true,"name":"var-log.mount"}]}}}}},"resources":{"infrastructureMachineTemplate":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachineTemplate","version":"v1beta2"},"infrastructureMachineTemplateSpecTemplateName":"controlplane-awsmachinetemplate-spec"}},"environmentVariables":{"hostName":"COREOS_EC2_HOSTNAME","ipv4":"COREOS_EC2_IPV4_LOCAL"},"kubeadmConfig":{"files":[{"contentFrom":{"secret":{"key":"kubelet-aws-config.sh","name":"provider-specific-files-4","prependClusterNameAsPrefix":true}},"path":"/opt/bin/kubelet-aws-config.sh","permissions":"0755"},{"contentFrom":{"secret":{"key":"kubelet-aws-config.service","name":"provider-specific-files-4","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/system/kubelet-aws-config.service","permissions":"0644"},{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files-4","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}],"ignition":{"containerLinuxConfig":{"additionalConfig":{"systemd":{"units":[{"enabled":true,"name":"kubelet-aws-config.service"}]}}}}},"osImage":{"variant":"3"},"pauseProperties":{"global.connectivity.vpcMode":"private"},"provider":"aws","registry":{"templateName":"awsContainerImageRegistry"},"resourcesApi":{"bastionResourceEnabled":false,"cleanupHelmReleaseResourcesEnabled":true,"clusterResourceEnabled":true,"controlPlaneResourceEnabled":true,"helmRepositoryResourcesEnabled":true,"infrastructureCluster":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSCluster","version":"v1beta2"},"infrastructureMachinePool":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachinePool","version":"v1beta2"},"machineHealthCheckResourceEnabled":true,"machinePoolResourcesEnabled":true,"nodePoolKind":"MachinePool"},"useReleases":true,"workers":{"defaultNodePools":{"def00":{"customNodeLabels":["label=default"],"instanceType":"r6i.xlarge","instanceWarmup":600,"maxSize":3,"minHealthyPercentage":90,"minSize":3}},"kubeadmConfig":{"files":[],"ignition":{"containerLinuxConfig":{"additionalConfig":{"storage":{"filesystems":[{"mount":{"device":"/dev/xvdd","format":"xfs","label":"lib","wipeFilesystem":true},"name":"lib"},{"mount":{"device":"/dev/xvde","format":"xfs","label":"log","wipeFilesystem":true},"name":"log"}]},"systemd":{"units":[{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/lib","where":"/var/lib"},"unit":{"defaultDependencies":false,"description":"lib volume"}},"enabled":true,"name":"var-lib.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/log","where":"/var/log"},"unit":{"defaultDependencies":false,"description":"log volume"}},"enabled":true,"name":"var-log.mount"}]}}}}}}}}`| +| `cluster` | **Cluster** - Helm values for the provider-independent cluster chart|**Type:** `object`
**Default:** `{"providerIntegration":{"apps":{"capiNodeLabeler":{"enable":true},"certExporter":{"enable":true},"certManager":{"enable":true},"chartOperatorExtensions":{"enable":true},"cilium":{"configTemplateName":"awsCiliumHelmValues","enable":true},"ciliumServiceMonitors":{"enable":true},"clusterAutoscaler":{"configTemplateName":"awsClusterAutoscalerHelmValues","enable":true},"coreDns":{"configTemplateName":"awsCorednsHelmValues","enable":true},"etcdKubernetesResourcesCountExporter":{"enable":true},"externalDns":{"configTemplateName":"awsExternalDnsHelmValues","enable":true},"k8sAuditMetrics":{"enable":true},"k8sDnsNodeCache":{"enable":true},"metricsServer":{"enable":true},"netExporter":{"enable":true},"networkPolicies":{"configTemplateName":"awsNetworkPoliciesHelmValues","enable":true},"nodeExporter":{"enable":true},"observabilityBundle":{"enable":true},"prometheusBlackboxExporter":{"enable":true},"securityBundle":{"configTemplateName":"awsSecurityBundleHelmValues","enable":true},"teleportKubeAgent":{"enable":true},"verticalPodAutoscaler":{"enable":true},"verticalPodAutoscalerCrd":{"enable":true}},"clusterAnnotationsTemplateName":"awsConnectivityLabels","components":{"systemd":{"timesyncd":{"ntp":["169.254.169.123"]}}},"connectivity":{"proxy":{"noProxy":{"templateName":"awsNoProxyList","value":["elb.amazonaws.com","169.254.169.254"]}}},"controlPlane":{"kubeadmConfig":{"clusterConfiguration":{"apiServer":{"apiAudiences":{"templateName":"awsApiServerApiAudiences"},"serviceAccountIssuers":[{"templateName":"awsIrsaServiceAccountIssuer"}]}},"files":[],"ignition":{"containerLinuxConfig":{"additionalConfig":{"storage":{"filesystems":[{"mount":{"device":"/dev/xvdc","format":"xfs","label":"etcd","wipeFilesystem":true},"name":"etcd"},{"mount":{"device":"/dev/xvdd","format":"xfs","label":"lib","wipeFilesystem":true},"name":"lib"},{"mount":{"device":"/dev/xvde","format":"xfs","label":"log","wipeFilesystem":true},"name":"log"}]},"systemd":{"units":[{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/etcd","where":"/var/lib/etcd"},"unit":{"defaultDependencies":false,"description":"etcd volume"}},"enabled":true,"name":"var-lib-etcd.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/lib","where":"/var/lib"},"unit":{"defaultDependencies":false,"description":"var lib volume"}},"enabled":true,"name":"var-lib.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/log","where":"/var/log"},"unit":{"defaultDependencies":false,"description":"log volume"}},"enabled":true,"name":"var-log.mount"}]}}}}},"resources":{"infrastructureMachineTemplate":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachineTemplate","version":"v1beta2"},"infrastructureMachineTemplateSpecTemplateName":"controlplane-awsmachinetemplate-spec"}},"environmentVariables":{"hostName":"COREOS_EC2_HOSTNAME","ipv4":"COREOS_EC2_IPV4_LOCAL"},"kubeadmConfig":{"files":[{"contentFrom":{"secret":{"key":"kubelet-aws-config.sh","name":"provider-specific-files-4","prependClusterNameAsPrefix":true}},"path":"/opt/bin/kubelet-aws-config.sh","permissions":"0755"},{"contentFrom":{"secret":{"key":"kubelet-aws-config.service","name":"provider-specific-files-4","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/system/kubelet-aws-config.service","permissions":"0644"},{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files-4","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}],"ignition":{"containerLinuxConfig":{"additionalConfig":{"systemd":{"units":[{"enabled":true,"name":"kubelet-aws-config.service"}]}}}}},"osImage":{"variant":"3"},"pauseProperties":{"global.connectivity.vpcMode":"private"},"provider":"aws","registry":{"templateName":"awsContainerImageRegistry"},"resourcesApi":{"bastionResourceEnabled":false,"cleanupHelmReleaseResourcesEnabled":true,"clusterResourceEnabled":true,"controlPlaneResourceEnabled":true,"helmRepositoryResourcesEnabled":true,"infrastructureCluster":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSCluster","version":"v1beta2"},"infrastructureMachinePool":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachinePool","version":"v1beta2"},"machineHealthCheckResourceEnabled":true,"machinePoolResourcesEnabled":true,"nodePoolKind":"MachinePool"},"useReleases":true,"workers":{"defaultNodePools":{"def00":{"customNodeLabels":["label=default"],"instanceType":"r6i.xlarge","instanceWarmup":600,"maxSize":3,"minHealthyPercentage":90,"minSize":3}},"kubeadmConfig":{"files":[],"ignition":{"containerLinuxConfig":{"additionalConfig":{"storage":{"filesystems":[{"mount":{"device":"/dev/xvdd","format":"xfs","label":"lib","wipeFilesystem":true},"name":"lib"},{"mount":{"device":"/dev/xvde","format":"xfs","label":"log","wipeFilesystem":true},"name":"log"}]},"systemd":{"units":[{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/lib","where":"/var/lib"},"unit":{"defaultDependencies":false,"description":"lib volume"}},"enabled":true,"name":"var-lib.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/log","where":"/var/log"},"unit":{"defaultDependencies":false,"description":"log volume"}},"enabled":true,"name":"var-log.mount"}]}}}}}}}}`| | `cluster-shared` | **Library chart**|**Type:** `object`
| | `managementCluster` | **Management cluster** - Name of the Cluster API cluster managing this workload cluster.|**Type:** `string`
| | `provider` | **Cluster API provider name**|**Type:** `string`
| diff --git a/helm/cluster-aws/ci/test-multiple-service-account-issuers.yaml b/helm/cluster-aws/ci/test-multiple-service-account-issuers.yaml new file mode 100644 index 00000000..441dd231 --- /dev/null +++ b/helm/cluster-aws/ci/test-multiple-service-account-issuers.yaml @@ -0,0 +1,25 @@ +global: + release: + version: v27.0.0-alpha.1 + metadata: + name: test-wc + organization: "test" + connectivity: + baseDomain: example.com + providerSpecific: + region: "eu-west-1" + +cluster: + internal: + ephemeralConfiguration: + offlineTesting: + renderWithoutReleaseResource: true + + providerIntegration: + controlPlane: + kubeadmConfig: + clusterConfiguration: + apiServer: + serviceAccountIssuers: + - url: https://first-issuer.example.org + - clusterDomainPrefix: second-issuer diff --git a/helm/cluster-aws/templates/_aws_cluster.tpl b/helm/cluster-aws/templates/_aws_cluster.tpl index 4cc74f33..c4d54d39 100644 --- a/helm/cluster-aws/templates/_aws_cluster.tpl +++ b/helm/cluster-aws/templates/_aws_cluster.tpl @@ -28,6 +28,10 @@ metadata: {{- if .Values.global.connectivity.topology.prefixListId }} network-topology.giantswarm.io/prefix-list: "{{ .Values.global.connectivity.topology.prefixListId }}" {{- end }} + {{- /* Used for migration from Vintage AWS to CAPA cluster. This adds all listed service account issuers to IAM trust policies. */}} + {{- if .Values.cluster.providerIntegration.controlPlane.kubeadmConfig.clusterConfiguration.apiServer.serviceAccountIssuers }} + aws.giantswarm.io/irsa-trust-domains: {{ include "service-account-issuers-comma-separated" $ | quote }} + {{- end }} labels: {{- include "labels.common" $ | nindent 4 }} {{- include "preventDeletionLabel" $ | nindent 4 -}} diff --git a/helm/cluster-aws/templates/_control_plane.tpl b/helm/cluster-aws/templates/_control_plane.tpl index f403c0b0..18833689 100644 --- a/helm/cluster-aws/templates/_control_plane.tpl +++ b/helm/cluster-aws/templates/_control_plane.tpl @@ -50,14 +50,17 @@ subnet: {{- end }} {{- end }} +{{- define "service-account-issuers-comma-separated" }} +{{- range $serviceAccountIssuerIndex, $serviceAccountIssuer := .Values.cluster.providerIntegration.controlPlane.kubeadmConfig.clusterConfiguration.apiServer.serviceAccountIssuers }} +{{- if gt $serviceAccountIssuerIndex 0 }},{{- end -}} +{{ regexReplaceAll "^(http://|https://)" (include "cluster.internal.controlPlane.kubeadm.clusterConfiguration.apiServer.serviceAccountIssuer" (dict "Values" $.Values "Release" $.Release "serviceAccountIssuer" $serviceAccountIssuer)) "" }} +{{- end -}} +{{- end -}} + {{- define "control-plane" }} apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: - annotations: - {{- if .Values.internal.migration.irsaAdditionalDomain }} - aws.giantswarm.io/irsa-additional-domain: "{{ .Values.internal.migration.irsaAdditionalDomain }}" - {{- end }} labels: cluster.x-k8s.io/role: control-plane {{- include "labels.common" $ | nindent 4 }} diff --git a/helm/cluster-aws/values.schema.json b/helm/cluster-aws/values.schema.json index 7beb9d2a..acbaab22 100644 --- a/helm/cluster-aws/values.schema.json +++ b/helm/cluster-aws/values.schema.json @@ -359,9 +359,11 @@ "apiAudiences": { "templateName": "awsApiServerApiAudiences" }, - "serviceAccountIssuer": { - "templateName": "awsIrsaServiceAccountIssuer" - } + "serviceAccountIssuers": [ + { + "templateName": "awsIrsaServiceAccountIssuer" + } + ] } }, "files": [], @@ -1722,18 +1724,6 @@ "type": "string", "title": "Hash salt", "description": "If specified, this token is used as a salt to the hash suffix of some resource names. Can be used to force-recreate some resources." - }, - "migration": { - "type": "object", - "title": "Migration values", - "description": "Section used for migration of cluster from vintage to CAPI", - "properties": { - "irsaAdditionalDomain": { - "type": "string", - "title": "IRSA additional domain", - "description": "Additional domain to be added to IRSA trust relationship." - } - } } } }, diff --git a/helm/cluster-aws/values.yaml b/helm/cluster-aws/values.yaml index 83d345ee..694a5052 100644 --- a/helm/cluster-aws/values.yaml +++ b/helm/cluster-aws/values.yaml @@ -72,8 +72,8 @@ cluster: apiServer: apiAudiences: templateName: awsApiServerApiAudiences - serviceAccountIssuer: - templateName: awsIrsaServiceAccountIssuer + serviceAccountIssuers: + - templateName: awsIrsaServiceAccountIssuer files: [] ignition: containerLinuxConfig: @@ -377,8 +377,7 @@ global: instanceMetadataOptions: httpTokens: required release: {} -internal: - migration: {} +internal: {} kubectlImage: name: giantswarm/kubectl registry: gsoci.azurecr.io