From 37b44430fc262c2a205af108e1b4f11a18afde86 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Sun, 29 Oct 2023 08:09:20 +0100 Subject: [PATCH] feat(karpenter): add karpenter configuration Signed-off-by: Christopher Haar --- Makefile | 4 +-- apis/composition.yaml | 51 +++++++++++++++----------------------- apis/definition.yaml | 2 -- crossplane.yaml | 4 +-- examples/eks-xr.yaml | 9 ++++--- examples/karpenter-xr.yaml | 5 ++-- examples/network-xr.yaml | 8 ++++++ test/karpenter-xr.yaml | 8 ++++++ test/setup.sh | 35 ++++++++++++++++++++++++++ 9 files changed, 82 insertions(+), 44 deletions(-) create mode 100644 examples/network-xr.yaml create mode 100644 test/karpenter-xr.yaml diff --git a/Makefile b/Makefile index 0563e15..c2df451 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ UPTEST_VERSION = v0.6.1 # ==================================================================================== # Setup XPKG XPKG_DIR = $(shell pwd) -XPKG_IGNORE = .github/workflows/*.yaml,.github/workflows/*.yml,examples/*.yaml,.work/uptest-datasource.yaml +XPKG_IGNORE = .github/workflows/*.yaml,.github/workflows/*.yml,examples/*.yaml,.work/uptest-datasource.yaml,test/*.yaml XPKG_REG_ORGS ?= xpkg.upbound.io/upbound # NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are # inferred. @@ -65,7 +65,7 @@ build.init: $(UP) # - UPTEST_DATASOURCE_PATH (optional), see https://github.com/upbound/uptest#injecting-dynamic-values-and-datasource uptest: $(UPTEST) $(KUBECTL) $(KUTTL) @$(INFO) running automated tests - @KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) $(UPTEST) e2e examples/karpenter-xr.yaml,examples/eks-xr.yaml --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL) + @KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) $(UPTEST) e2e test/karpenter-xr.yaml,examples/network-xr.yaml,examples/eks-xr.yaml --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL) @$(OK) running automated tests # This target requires the following environment variables to be set: diff --git a/apis/composition.yaml b/apis/composition.yaml index a4d0d6a..2b56ae6 100644 --- a/apis/composition.yaml +++ b/apis/composition.yaml @@ -26,7 +26,7 @@ spec: fromFieldPath: spec.parameters.region toFieldPath: spec.forProvider.region resources: - - name: InstanceNodeRole + - name: instanceNodeRole base: apiVersion: iam.aws.upbound.io/v1beta1 kind: Role @@ -57,13 +57,19 @@ spec: patchSetName: providerConfigRef - type: PatchSet patchSetName: deletionPolicy + - fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: string + string: + fmt: "KarpenterNodeRole-%s" - type: ToCompositeFieldPath fromFieldPath: status.atProvider.arn toFieldPath: status.karpenter.instanceProfileRoleArn policy: fromFieldPath: Optional - type: ToCompositeFieldPath - fromFieldPath: status.roleArn + fromFieldPath: status.atProvider.arn toFieldPath: status.karpenter.accountId transforms: - type: string @@ -73,7 +79,7 @@ spec: match: '::(\d+):' group: 1 - - name: InstanceNodeRoleEKSPolicy + - name: instanceNodeRoleEKSPolicy base: apiVersion: iam.aws.upbound.io/v1beta1 kind: RolePolicyAttachment @@ -107,7 +113,7 @@ spec: - type: PatchSet patchSetName: deletionPolicy - - name: InstanceNodeRoleECRPolicy + - name: instanceNodeRoleECRPolicy base: apiVersion: iam.aws.upbound.io/v1beta1 kind: RolePolicyAttachment @@ -124,7 +130,7 @@ spec: - type: PatchSet patchSetName: deletionPolicy - - name: InstanceNodeRoleSSMPolicy + - name: instanceNodeRoleSSMPolicy base: apiVersion: iam.aws.upbound.io/v1beta1 kind: RolePolicyAttachment @@ -141,7 +147,7 @@ spec: - type: PatchSet patchSetName: deletionPolicy - - name: InstanceProfile + - name: instanceProfile base: apiVersion: iam.aws.upbound.io/v1beta1 kind: InstanceProfile @@ -169,7 +175,7 @@ spec: - name: IRSA base: - apiVersion: aws.platformref.upbound.io/v1alpha1 # ToDo(haarchri): change apiVersion when split is done! + apiVersion: aws.platform.upbound.io/v1alpha1 kind: XIRSA spec: parameters: @@ -196,6 +202,8 @@ spec: policy: fromFieldPath: Optional - type: CombineFromComposite + policy: + fromFieldPath: Required combine: variables: - fromFieldPath: spec.parameters.region @@ -427,6 +435,8 @@ spec: - type: PatchSet patchSetName: region - type: CombineFromComposite + policy: + fromFieldPath: Required combine: variables: - fromFieldPath: status.karpenter.sqsQueueArn @@ -750,6 +760,7 @@ spec: apiVersion: helm.crossplane.io/v1beta1 kind: Release spec: + deletionPolicy: Orphan forProvider: namespace: karpenter chart: @@ -761,8 +772,6 @@ spec: aws: nodeNameConvention: ip-name patches: - - type: PatchSet - patchSetName: deletionPolicy - fromFieldPath: spec.parameters.id toFieldPath: spec.providerConfigRef.name - fromFieldPath: status.karpenter.IRSARoleArn @@ -794,6 +803,7 @@ spec: apiVersion: kubernetes.crossplane.io/v1alpha1 kind: Object spec: + deletionPolicy: Orphan forProvider: manifest: apiVersion: karpenter.sh/v1alpha5 @@ -828,8 +838,6 @@ spec: providerRef: name: default patches: - - type: PatchSet - patchSetName: deletionPolicy - fromFieldPath: spec.parameters.id toFieldPath: spec.providerConfigRef.name @@ -838,6 +846,7 @@ spec: apiVersion: kubernetes.crossplane.io/v1alpha1 kind: Object spec: + deletionPolicy: Orphan forProvider: manifest: apiVersion: karpenter.k8s.aws/v1alpha1 @@ -850,8 +859,6 @@ spec: NodeType: "default" intent: apps patches: - - type: PatchSet - patchSetName: deletionPolicy - fromFieldPath: spec.parameters.id toFieldPath: spec.providerConfigRef.name - fromFieldPath: spec.parameters.id @@ -864,21 +871,3 @@ spec: toFieldPath: spec.forProvider.manifest.spec.instanceProfile policy: fromFieldPath: Required - - # ToDo(haarchri): add usages for helm-chart and objects - # - name: XEKSUsagesChart - # base: - # apiVersion: apiextensions.crossplane.io/v1alpha1 - # kind: Usage - # spec: - # of: - # apiVersion: aws.platform.upbound.io/v1alpha1 - # kind: XEKS - # by: - # apiVersion: helm.crossplane.io/v1beta1 - # kind: Release - # resourceSelector: - # matchControllerRef: true - # patches: - # - fromFieldPath: spec.parameters.providerConfigName - # toFieldPath: spec.of.resourceSelector.matchLabels[xeks.aws.platform.upbound.io/cluster-id] \ No newline at end of file diff --git a/apis/definition.yaml b/apis/definition.yaml index 27c23a5..4bbc92d 100644 --- a/apis/definition.yaml +++ b/apis/definition.yaml @@ -3,8 +3,6 @@ kind: CompositeResourceDefinition metadata: name: xkarpenters.aws.platform.upbound.io spec: - connectionSecretKeys: - - kubeconfig group: aws.platform.upbound.io names: kind: XKarpenter diff --git a/crossplane.yaml b/crossplane.yaml index 32c5a5f..cf02ac1 100644 --- a/crossplane.yaml +++ b/crossplane.yaml @@ -10,9 +10,7 @@ spec: crossplane: version: ">=v1.13.2-0" dependsOn: - - configuration: xpkg.upbound.io/upbound/configuration-aws-eks - version: ">=v0.1.0" - - configuration: xpkg.upbound.io/upbound/configuration-aws-irsa + - configuration: xpkg.upbound.io/upbound/configuration-aws-eks-irsa version: ">=v0.1.0" - provider: xpkg.upbound.io/upbound/provider-aws-sqs version: ">=v0.36.0" diff --git a/examples/eks-xr.yaml b/examples/eks-xr.yaml index 23b6cd3..65bd884 100644 --- a/examples/eks-xr.yaml +++ b/examples/eks-xr.yaml @@ -1,19 +1,20 @@ apiVersion: aws.platform.upbound.io/v1alpha1 kind: XEKS metadata: - name: configuration-aws-eks + name: configuration-aws-eks-karpenter + labels: + xeks.aws.platform.upbound.io/cluster-id: configuration-aws-eks-karpenter spec: parameters: - id: configuration-aws-eks + id: configuration-aws-eks-karpenter region: us-west-2 version: "1.27" iam: # replace with your custom arn like: - # roleArn: arn:aws:iam::123456789:role/AWSReservedSSO_AdministratorAccess_d703c73ed340fde7 roleArn: ${data.aws_eks_iam_default_admin} nodes: count: 3 instanceType: t3.small writeConnectionSecretToRef: - name: configuration-aws-eks-kubeconfig + name: configuration-aws-eks-karpenter-kubeconfig namespace: upbound-system diff --git a/examples/karpenter-xr.yaml b/examples/karpenter-xr.yaml index 50f5140..6f10ecb 100644 --- a/examples/karpenter-xr.yaml +++ b/examples/karpenter-xr.yaml @@ -4,6 +4,7 @@ metadata: name: configuration-aws-eks-karpenter spec: parameters: - clusterName: configuration-aws-eks-rktp2 - id: configuration-aws-eks + # replace with your clusterName + clusterName: + id: configuration-aws-eks-karpenter region: us-west-2 diff --git a/examples/network-xr.yaml b/examples/network-xr.yaml new file mode 100644 index 0000000..6561e81 --- /dev/null +++ b/examples/network-xr.yaml @@ -0,0 +1,8 @@ +apiVersion: aws.platform.upbound.io/v1alpha1 +kind: XNetwork +metadata: + name: configuration-aws-eks-karpenter +spec: + parameters: + id: configuration-aws-eks-karpenter + region: us-west-2 diff --git a/test/karpenter-xr.yaml b/test/karpenter-xr.yaml new file mode 100644 index 0000000..f88d196 --- /dev/null +++ b/test/karpenter-xr.yaml @@ -0,0 +1,8 @@ +apiVersion: aws.platform.upbound.io/v1alpha1 +kind: XKarpenter +metadata: + name: configuration-aws-eks-karpenter +spec: + parameters: + id: configuration-aws-eks-karpenter + region: us-west-2 diff --git a/test/setup.sh b/test/setup.sh index ab6516a..74d48b6 100755 --- a/test/setup.sh +++ b/test/setup.sh @@ -33,3 +33,38 @@ spec: namespace: upbound-system source: Secret EOF + +SCRIPT_DIR=$( cd -- $( dirname -- "${BASH_SOURCE[0]}" ) &> /dev/null && pwd ) + +"${KUBECTL}" apply -f ${SCRIPT_DIR}/../examples/eks-xr.yaml + +# Function to extract the annotation from a resource +get_annotation() { + local resource_json="$1" + local annotation="$2" + annotation_value=$(echo "$resource_json" | grep -o "\"$annotation\": \"[^\"]*\"" | cut -d '"' -f 4) + echo "$annotation_value" +} + +# Watch for changes to the resource and extract the annotation +while true; do + resource_info=$(kubectl get cluster.eks.aws.upbound.io -o json) + annotation_value=$(get_annotation "$resource_info" "crossplane.io/external-name") + + if [ -n "$annotation_value" ]; then + cat <