From 6e4e367a657d41b0ff0b43495d81dd01a9a1a706 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Sun, 29 Oct 2023 08:09:41 +0100 Subject: [PATCH] feat(karpenter): add karpenter configuration Signed-off-by: Christopher Haar --- Makefile | 6 +- README.md | 2 +- apis/composition.yaml | 873 ++++++++++++++++++++++++++++++++++++ apis/definition.yaml | 60 +++ crossplane.yaml | 18 + examples/configuration.yaml | 6 + examples/eks-xr.yaml | 20 + examples/karpenter-xr.yaml | 10 + examples/network-xr.yaml | 8 + test/karpenter-xr.yaml | 8 + test/setup.sh | 70 +++ test/workload.yaml | 25 ++ 12 files changed, 1102 insertions(+), 4 deletions(-) create mode 100644 apis/composition.yaml create mode 100644 apis/definition.yaml create mode 100644 crossplane.yaml create mode 100644 examples/configuration.yaml create mode 100644 examples/eks-xr.yaml create mode 100644 examples/karpenter-xr.yaml create mode 100644 examples/network-xr.yaml create mode 100644 test/karpenter-xr.yaml create mode 100755 test/setup.sh create mode 100644 test/workload.yaml diff --git a/Makefile b/Makefile index 2368758..c2df451 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ UPTEST_VERSION = v0.6.1 # ==================================================================================== # Setup XPKG XPKG_DIR = $(shell pwd) -XPKG_IGNORE = .github/workflows/*.yaml,.github/workflows/*.yml,examples/*.yaml,.work/uptest-datasource.yaml +XPKG_IGNORE = .github/workflows/*.yaml,.github/workflows/*.yml,examples/*.yaml,.work/uptest-datasource.yaml,test/*.yaml XPKG_REG_ORGS ?= xpkg.upbound.io/upbound # NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are # inferred. @@ -65,11 +65,11 @@ build.init: $(UP) # - UPTEST_DATASOURCE_PATH (optional), see https://github.com/upbound/uptest#injecting-dynamic-values-and-datasource uptest: $(UPTEST) $(KUBECTL) $(KUTTL) @$(INFO) running automated tests - @KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) $(UPTEST) e2e examples/eks-xr.yaml,examples/karpenter-xr.yaml --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL) + @KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) $(UPTEST) e2e test/karpenter-xr.yaml,examples/network-xr.yaml,examples/eks-xr.yaml --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL) @$(OK) running automated tests # This target requires the following environment variables to be set: # - UPTEST_CLOUD_CREDENTIALS, cloud credentials for the provider being tested, e.g. export UPTEST_CLOUD_CREDENTIALS=$(cat ~/.aws/credentials) e2e: build controlplane.up local.xpkg.deploy.configuration.$(PROJECT_NAME) uptest -.PHONY: uptest e2e \ No newline at end of file +.PHONY: uptest e2e diff --git a/README.md b/README.md index fda9a52..7ccdf8d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ # AWS EKS Karpenter Configuration AWS EKS Karpenter Configuration is reusable Configuration designed to be primarily used in -higher level Configurations. \ No newline at end of file +higher level Configurations. diff --git a/apis/composition.yaml b/apis/composition.yaml new file mode 100644 index 0000000..2b56ae6 --- /dev/null +++ b/apis/composition.yaml @@ -0,0 +1,873 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: xkarpenters.aws.platform.upbound.io + labels: + provider: aws +spec: + writeConnectionSecretsToNamespace: upbound-system + compositeTypeRef: + apiVersion: aws.platform.upbound.io/v1alpha1 + kind: XKarpenter + patchSets: + - name: providerConfigRef + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.providerConfigName + toFieldPath: spec.providerConfigRef.name + - name: deletionPolicy + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.deletionPolicy + toFieldPath: spec.deletionPolicy + - name: region + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.region + toFieldPath: spec.forProvider.region + resources: + - name: instanceNodeRole + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: Role + metadata: + labels: + role: karpenter + spec: + forProvider: + assumeRolePolicy: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": [ + "ec2.amazonaws.com" + ] + }, + "Action": [ + "sts:AssumeRole" + ] + } + ] + } + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: string + string: + fmt: "KarpenterNodeRole-%s" + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.arn + toFieldPath: status.karpenter.instanceProfileRoleArn + policy: + fromFieldPath: Optional + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.arn + toFieldPath: status.karpenter.accountId + transforms: + - type: string + string: + type: Regexp + regexp: + match: '::(\d+):' + group: 1 + + - name: instanceNodeRoleEKSPolicy + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy + roleSelector: + matchControllerRef: true + matchLabels: + role: karpenter + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + + - name: InstanceNodeRoleCNIPolicy + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy + roleSelector: + matchControllerRef: true + matchLabels: + role: karpenter + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + + - name: instanceNodeRoleECRPolicy + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly + roleSelector: + matchControllerRef: true + matchLabels: + role: karpenter + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + + - name: instanceNodeRoleSSMPolicy + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore + roleSelector: + matchControllerRef: true + matchLabels: + role: karpenter + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + + - name: instanceProfile + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: InstanceProfile + spec: + forProvider: + roleSelector: + matchControllerRef: true + matchLabels: + role: karpenter + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.arn + toFieldPath: status.karpenter.instanceProfileArn + policy: + fromFieldPath: Optional + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.id + toFieldPath: status.karpenter.instanceProfileName + policy: + fromFieldPath: Optional + + - name: IRSA + base: + apiVersion: aws.platform.upbound.io/v1alpha1 + kind: XIRSA + spec: + parameters: + condition: StringEquals + serviceAccount: + name: karpenter + namespace: karpenter + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.name + transforms: + - type: string + string: + fmt: "%s-karpenter" + - fromFieldPath: spec.parameters.id + toFieldPath: spec.parameters.id + - type: ToCompositeFieldPath + fromFieldPath: status.roleArn + toFieldPath: status.karpenter.IRSARoleArn + policy: + fromFieldPath: Optional + - type: CombineFromComposite + policy: + fromFieldPath: Required + combine: + variables: + - fromFieldPath: spec.parameters.region + - fromFieldPath: spec.parameters.clusterName + - fromFieldPath: status.karpenter.sqsQueueArn + - fromFieldPath: status.karpenter.accountId + - fromFieldPath: status.karpenter.instanceProfileRoleArn + strategy: string + string: + fmt: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowScopedEC2InstanceActions", + "Effect": "Allow", + "Resource": [ + "arn:aws:ec2:%[1]s::image/*", + "arn:aws:ec2:%[1]s::snapshot/*", + "arn:aws:ec2:%[1]s:*:spot-instances-request/*", + "arn:aws:ec2:%[1]s:*:security-group/*", + "arn:aws:ec2:%[1]s:*:subnet/*", + "arn:aws:ec2:%[1]s:*:launch-template/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + }, + { + "Sid": "AllowScopedEC2InstanceActionsWithTags", + "Effect": "Allow", + "Resource": [ + "arn:aws:ec2:%[1]s:*:fleet/*", + "arn:aws:ec2:%[1]s:*:instance/*", + "arn:aws:ec2:%[1]s:*:volume/*", + "arn:aws:ec2:%[1]s:*:network-interface/*", + "arn:aws:ec2:%[1]s:*:launch-template/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/%[2]s": "owned" + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/provisioner-name": "*" + } + } + }, + { + "Sid": "AllowScopedResourceCreationTagging", + "Effect": "Allow", + "Resource": [ + "arn:aws:ec2:%[1]s:*:fleet/*", + "arn:aws:ec2:%[1]s:*:instance/*", + "arn:aws:ec2:%[1]s:*:volume/*", + "arn:aws:ec2:%[1]s:*:network-interface/*", + "arn:aws:ec2:%[1]s:*:launch-template/*" + ], + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/%[2]s": "owned", + "ec2:CreateAction": [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate" + ] + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/provisioner-name": "*" + } + } + }, + { + "Sid": "AllowMachineMigrationTagging", + "Effect": "Allow", + "Resource": "arn:aws:ec2:%[1]s:*:instance/*", + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/%[2]s": "owned", + "aws:RequestTag/karpenter.sh/managed-by": "%[2]s" + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/provisioner-name": "*" + }, + "ForAllValues:StringEquals": { + "aws:TagKeys": [ + "karpenter.sh/provisioner-name", + "karpenter.sh/managed-by" + ] + } + } + }, + { + "Sid": "AllowScopedDeletion", + "Effect": "Allow", + "Resource": [ + "arn:aws:ec2:%[1]s:*:instance/*", + "arn:aws:ec2:%[1]s:*:launch-template/*" + ], + "Action": [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/%[2]s": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/provisioner-name": "*" + } + } + }, + { + "Sid": "AllowRegionalReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ], + "Condition": { + "StringEquals": { + "aws:RequestedRegion": "%[1]s" + } + } + }, + { + "Sid": "AllowSSMReadActions", + "Effect": "Allow", + "Resource": "arn:aws:ssm:%[1]s::parameter/aws/service/*", + "Action": "ssm:GetParameter" + }, + { + "Sid": "AllowPricingReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "pricing:GetProducts" + }, + { + "Sid": "AllowInterruptionQueueActions", + "Effect": "Allow", + "Resource": "%[3]s", + "Action": [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + }, + { + "Sid": "AllowPassingInstanceRole", + "Effect": "Allow", + "Resource": "%[5]s", + "Action": "iam:PassRole", + "Condition": { + "StringEquals": { + "iam:PassedToService": "ec2.amazonaws.com" + } + } + }, + { + "Sid": "AllowAPIServerEndpointDiscovery", + "Effect": "Allow", + "Resource": "arn:aws:eks:%[1]s:%[4]s:cluster/%[2]s", + "Action": "eks:DescribeCluster" + } + ] + } + toFieldPath: spec.parameters.policyDocument + + - name: sqsQueue + base: + apiVersion: sqs.aws.upbound.io/v1beta1 + kind: Queue + spec: + forProvider: + messageRetentionSeconds: 300 + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + - fromFieldPath: spec.parameters.id + toFieldPath: spec.forProvider.name + transforms: + - type: string + string: + fmt: "%s-karpenter" + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.arn + toFieldPath: status.karpenter.sqsQueueArn + policy: + fromFieldPath: Optional + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.name + toFieldPath: status.karpenter.sqsQueueName + policy: + fromFieldPath: Optional + + - name: sqsQueuePolicy + base: + apiVersion: sqs.aws.upbound.io/v1beta1 + kind: QueuePolicy + spec: + forProvider: + queueUrlSelector: + matchControllerRef: true + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + - type: CombineFromComposite + policy: + fromFieldPath: Required + combine: + variables: + - fromFieldPath: status.karpenter.sqsQueueArn + strategy: string + string: + fmt: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "SqsWrite", + "Effect": "Allow", + "Principal": "*", + "Action": "sqs:SendMessage", + "Resource": "%s", + "Principal": { + "Service": [ + "events.amazonaws.com", + "sqs.amazonaws.com" + ] + } + } + ] + } + toFieldPath: spec.forProvider.policy + + - name: ruleHealthEvent + base: + apiVersion: cloudwatchevents.aws.upbound.io/v1beta1 + kind: Rule + metadata: + labels: + type: HealthEvent + spec: + forProvider: + # https://github.com/upbound/provider-aws/issues/931 + eventBusName: default + eventPattern: | + { + "source": [ + "aws.health" + ], + "detail-type": [ + "AWS Health Event" + ] + } + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + # https://github.com/upbound/provider-aws/issues/931 + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: '.*' + result: healthevent + # https://github.com/upbound/provider-aws/issues/931 + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.name + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: '.*' + result: healthevent + + - name: ruleSpotInterrupt + base: + apiVersion: cloudwatchevents.aws.upbound.io/v1beta1 + kind: Rule + metadata: + labels: + type: SpotInterrupt + spec: + forProvider: + # https://github.com/upbound/provider-aws/issues/931 + eventBusName: default + eventPattern: | + { + "source": [ + "aws.ec2" + ], + "detail-type": [ + "EC2 Spot Instance Interruption Warning" + ] + } + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + # https://github.com/upbound/provider-aws/issues/931 + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: '.*' + result: spotinterrupt + # https://github.com/upbound/provider-aws/issues/931 + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.name + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: '.*' + result: spotinterrupt + + - name: ruleInstanceRebalance + base: + apiVersion: cloudwatchevents.aws.upbound.io/v1beta1 + kind: Rule + metadata: + labels: + type: InstanceRebalance + spec: + forProvider: + # https://github.com/upbound/provider-aws/issues/931 + eventBusName: default + eventPattern: | + { + "source": [ + "aws.ec2" + ], + "detail-type": [ + "EC2 Instance Rebalance Recommendation" + ] + } + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + # https://github.com/upbound/provider-aws/issues/931 + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: '.*' + result: instancerebalance + # https://github.com/upbound/provider-aws/issues/931 + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.name + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: '.*' + result: instancerebalance + + - name: ruleInstanceStateChange + base: + apiVersion: cloudwatchevents.aws.upbound.io/v1beta1 + kind: Rule + metadata: + labels: + type: InstanceStateChange + spec: + forProvider: + # https://github.com/upbound/provider-aws/issues/931 + eventBusName: default + eventPattern: | + { + "source": [ + "aws.ec2" + ], + "detail-type": [ + "EC2 Instance State-change Notification" + ] + } + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + # https://github.com/upbound/provider-aws/issues/931 + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: '.*' + result: instancestatechange + # https://github.com/upbound/provider-aws/issues/931 + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.name + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: '.*' + result: instancestatechange + + - name: ruleHealthEventTarget + base: + apiVersion: cloudwatchevents.aws.upbound.io/v1beta1 + kind: Target + spec: + forProvider: + # https://github.com/upbound/provider-aws/issues/932 + eventBusName: default + ruleSelector: + matchControllerRef: true + matchLabels: + type: HealthEvent + targetId: healthevent + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + - fromFieldPath: status.karpenter.sqsQueueArn + toFieldPath: spec.forProvider.arn + policy: + fromFieldPath: Required + + - name: ruleSpotInterruptTarget + base: + apiVersion: cloudwatchevents.aws.upbound.io/v1beta1 + kind: Target + spec: + forProvider: + # https://github.com/upbound/provider-aws/issues/932 + eventBusName: default + ruleSelector: + matchControllerRef: true + matchLabels: + type: SpotInterrupt + targetId: spotinterrupt + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + - fromFieldPath: status.karpenter.sqsQueueArn + toFieldPath: spec.forProvider.arn + policy: + fromFieldPath: Required + + - name: ruleInstanceRebalanceTarget + base: + apiVersion: cloudwatchevents.aws.upbound.io/v1beta1 + kind: Target + spec: + forProvider: + # https://github.com/upbound/provider-aws/issues/932 + eventBusName: default + ruleSelector: + matchControllerRef: true + matchLabels: + type: InstanceRebalance + targetId: instancerebalance + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + - fromFieldPath: status.karpenter.sqsQueueArn + toFieldPath: spec.forProvider.arn + policy: + fromFieldPath: Required + + - name: ruleInstanceStateChangeTarget + base: + apiVersion: cloudwatchevents.aws.upbound.io/v1beta1 + kind: Target + spec: + forProvider: + # https://github.com/upbound/provider-aws/issues/932 + eventBusName: default + ruleSelector: + matchControllerRef: true + matchLabels: + type: InstanceStateChange + targetId: instancestatechange + patches: + - type: PatchSet + patchSetName: providerConfigRef + - type: PatchSet + patchSetName: deletionPolicy + - type: PatchSet + patchSetName: region + - fromFieldPath: status.karpenter.sqsQueueArn + toFieldPath: spec.forProvider.arn + policy: + fromFieldPath: Required + + - name: karpenterChart + base: + apiVersion: helm.crossplane.io/v1beta1 + kind: Release + spec: + deletionPolicy: Orphan + forProvider: + namespace: karpenter + chart: + name: karpenter + repository: oci://public.ecr.aws/karpenter + version: v0.31.1 + values: + settings: + aws: + nodeNameConvention: ip-name + patches: + - fromFieldPath: spec.parameters.id + toFieldPath: spec.providerConfigRef.name + - fromFieldPath: status.karpenter.IRSARoleArn + toFieldPath: spec.forProvider.values.serviceAccount.annotations[eks.amazonaws.com/role-arn] + policy: + fromFieldPath: Required + - fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.values.settings.aws.clusterName + - fromFieldPath: status.karpenter.instanceProfileName + toFieldPath: spec.forProvider.values.settings.aws.defaultInstanceProfile + policy: + fromFieldPath: Required + - fromFieldPath: status.karpenter.sqsQueueName + toFieldPath: spec.forProvider.values.settings.aws.interruptionQueueName + policy: + fromFieldPath: Required + - fromFieldPath: spec.parameters.id + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: '.*' + result: karpenter + + - name: karpenterProvisioner + base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + deletionPolicy: Orphan + forProvider: + manifest: + apiVersion: karpenter.sh/v1alpha5 + kind: Provisioner + metadata: + name: default + spec: + labels: + intent: apps + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r", "i", "d"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32", "48", "64"] + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["amd64", "arm64"] + kubeletConfiguration: + containerRuntime: containerd + limits: + resources: + cpu: 1000 + memory: 500Gi + consolidation: + enabled: true + ttlSecondsUntilExpired: 604800 + providerRef: + name: default + patches: + - fromFieldPath: spec.parameters.id + toFieldPath: spec.providerConfigRef.name + + - name: karpenterAWSNodeTemplate + base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + deletionPolicy: Orphan + forProvider: + manifest: + apiVersion: karpenter.k8s.aws/v1alpha1 + kind: AWSNodeTemplate + metadata: + name: default + spec: + tags: + KarpenterProvisionerName: "default" + NodeType: "default" + intent: apps + patches: + - fromFieldPath: spec.parameters.id + toFieldPath: spec.providerConfigRef.name + - fromFieldPath: spec.parameters.id + toFieldPath: spec.forProvider.manifest.spec.subnetSelector[networks.aws.platform.upbound.io/network-id] + - fromFieldPath: spec.parameters.id + toFieldPath: spec.forProvider.manifest.spec.securityGroupSelector[eks.aws.platform.upbound.io/discovery] + - fromFieldPath: spec.parameters.id + toFieldPath: spec.forProvider.manifest.spec.tags[karpenter.sh/discovery] + - fromFieldPath: status.karpenter.instanceProfileName + toFieldPath: spec.forProvider.manifest.spec.instanceProfile + policy: + fromFieldPath: Required diff --git a/apis/definition.yaml b/apis/definition.yaml new file mode 100644 index 0000000..4bbc92d --- /dev/null +++ b/apis/definition.yaml @@ -0,0 +1,60 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: xkarpenters.aws.platform.upbound.io +spec: + group: aws.platform.upbound.io + names: + kind: XKarpenter + plural: xkarpenters + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + parameters: + type: object + description: EKS configuration parameters. + properties: + id: + type: string + description: ID of this Cluster that other objects will use to refer to it. + clusterName: + type: string + description: Full Name of Cluster. + region: + type: string + description: Region is the region you'd like your resource to be created in. + deletionPolicy: + description: Delete the external resources when the Claim/XR is deleted. Defaults to Delete + enum: + - Delete + - Orphan + type: string + default: Delete + providerConfigName: + description: Crossplane ProviderConfig to use for provisioning this resources + type: string + default: default + required: + - id + - clusterName + - region + - deletionPolicy + - providerConfigName + required: + - parameters + status: + description: A Status represents the observed state + properties: + karpenter: + description: Freeform field containing status information for karpenter + type: object + x-kubernetes-preserve-unknown-fields: true + type: object diff --git a/crossplane.yaml b/crossplane.yaml new file mode 100644 index 0000000..cf02ac1 --- /dev/null +++ b/crossplane.yaml @@ -0,0 +1,18 @@ +apiVersion: meta.pkg.crossplane.io/v1alpha1 +kind: Configuration +metadata: + name: configuration-aws-eks-karpenter + annotations: + meta.crossplane.io/maintainer: Upbound + meta.crossplane.io/source: github.com/upbound/configuration-aws-eks-karpenter + meta.crossplane.io/license: Apache-2.0 +spec: + crossplane: + version: ">=v1.13.2-0" + dependsOn: + - configuration: xpkg.upbound.io/upbound/configuration-aws-eks-irsa + version: ">=v0.1.0" + - provider: xpkg.upbound.io/upbound/provider-aws-sqs + version: ">=v0.36.0" + - provider: xpkg.upbound.io/upbound/provider-aws-cloudwatchevents + version: ">=v0.36.0" diff --git a/examples/configuration.yaml b/examples/configuration.yaml new file mode 100644 index 0000000..5ee89e5 --- /dev/null +++ b/examples/configuration.yaml @@ -0,0 +1,6 @@ +apiVersion: pkg.crossplane.io/v1 +kind: Configuration +metadata: + name: cofiguration-aws-eks-karpenter +spec: + package: xpkg.upbound.io/upbound/configuration-aws-eks-karpenter:v0.1.0 diff --git a/examples/eks-xr.yaml b/examples/eks-xr.yaml new file mode 100644 index 0000000..65bd884 --- /dev/null +++ b/examples/eks-xr.yaml @@ -0,0 +1,20 @@ +apiVersion: aws.platform.upbound.io/v1alpha1 +kind: XEKS +metadata: + name: configuration-aws-eks-karpenter + labels: + xeks.aws.platform.upbound.io/cluster-id: configuration-aws-eks-karpenter +spec: + parameters: + id: configuration-aws-eks-karpenter + region: us-west-2 + version: "1.27" + iam: + # replace with your custom arn like: + roleArn: ${data.aws_eks_iam_default_admin} + nodes: + count: 3 + instanceType: t3.small + writeConnectionSecretToRef: + name: configuration-aws-eks-karpenter-kubeconfig + namespace: upbound-system diff --git a/examples/karpenter-xr.yaml b/examples/karpenter-xr.yaml new file mode 100644 index 0000000..6f10ecb --- /dev/null +++ b/examples/karpenter-xr.yaml @@ -0,0 +1,10 @@ +apiVersion: aws.platform.upbound.io/v1alpha1 +kind: XKarpenter +metadata: + name: configuration-aws-eks-karpenter +spec: + parameters: + # replace with your clusterName + clusterName: + id: configuration-aws-eks-karpenter + region: us-west-2 diff --git a/examples/network-xr.yaml b/examples/network-xr.yaml new file mode 100644 index 0000000..6561e81 --- /dev/null +++ b/examples/network-xr.yaml @@ -0,0 +1,8 @@ +apiVersion: aws.platform.upbound.io/v1alpha1 +kind: XNetwork +metadata: + name: configuration-aws-eks-karpenter +spec: + parameters: + id: configuration-aws-eks-karpenter + region: us-west-2 diff --git a/test/karpenter-xr.yaml b/test/karpenter-xr.yaml new file mode 100644 index 0000000..f88d196 --- /dev/null +++ b/test/karpenter-xr.yaml @@ -0,0 +1,8 @@ +apiVersion: aws.platform.upbound.io/v1alpha1 +kind: XKarpenter +metadata: + name: configuration-aws-eks-karpenter +spec: + parameters: + id: configuration-aws-eks-karpenter + region: us-west-2 diff --git a/test/setup.sh b/test/setup.sh new file mode 100755 index 0000000..74d48b6 --- /dev/null +++ b/test/setup.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -aeuo pipefail + +echo "Running setup.sh" +echo "Waiting until all configurations are healthy/installed..." +"${KUBECTL}" wait configuration.pkg --all --for=condition=Healthy --timeout 5m +"${KUBECTL}" wait configuration.pkg --all --for=condition=Installed --timeout 5m + +echo "Creating cloud credential secret..." +"${KUBECTL}" -n upbound-system create secret generic aws-creds --from-literal=credentials="${UPTEST_CLOUD_CREDENTIALS}" \ + --dry-run=client -o yaml | "${KUBECTL}" apply -f - + +echo "Waiting until all installed provider packages are healthy..." +"${KUBECTL}" wait provider.pkg --all --for condition=Healthy --timeout 5m + +echo "Waiting for all pods to come online..." +"${KUBECTL}" -n upbound-system wait --for=condition=Available deployment --all --timeout=5m + +echo "Waiting for all XRDs to be established..." +"${KUBECTL}" wait xrd --all --for condition=Established + +echo "Creating a default provider config..." +cat < /dev/null && pwd ) + +"${KUBECTL}" apply -f ${SCRIPT_DIR}/../examples/eks-xr.yaml + +# Function to extract the annotation from a resource +get_annotation() { + local resource_json="$1" + local annotation="$2" + annotation_value=$(echo "$resource_json" | grep -o "\"$annotation\": \"[^\"]*\"" | cut -d '"' -f 4) + echo "$annotation_value" +} + +# Watch for changes to the resource and extract the annotation +while true; do + resource_info=$(kubectl get cluster.eks.aws.upbound.io -o json) + annotation_value=$(get_annotation "$resource_info" "crossplane.io/external-name") + + if [ -n "$annotation_value" ]; then + cat <