Skip to content

Commit

Permalink
feat(karpenter): add karpenter configuration
Browse files Browse the repository at this point in the history
Signed-off-by: Christopher Haar <[email protected]>
  • Loading branch information
haarchri committed Oct 29, 2023
1 parent 2e99618 commit 37b4443
Show file tree
Hide file tree
Showing 9 changed files with 82 additions and 44 deletions.
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ UPTEST_VERSION = v0.6.1
# ====================================================================================
# Setup XPKG
XPKG_DIR = $(shell pwd)
XPKG_IGNORE = .github/workflows/*.yaml,.github/workflows/*.yml,examples/*.yaml,.work/uptest-datasource.yaml
XPKG_IGNORE = .github/workflows/*.yaml,.github/workflows/*.yml,examples/*.yaml,.work/uptest-datasource.yaml,test/*.yaml
XPKG_REG_ORGS ?= xpkg.upbound.io/upbound
# NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are
# inferred.
Expand Down Expand Up @@ -65,7 +65,7 @@ build.init: $(UP)
# - UPTEST_DATASOURCE_PATH (optional), see https://github.com/upbound/uptest#injecting-dynamic-values-and-datasource
uptest: $(UPTEST) $(KUBECTL) $(KUTTL)
@$(INFO) running automated tests
@KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) $(UPTEST) e2e examples/karpenter-xr.yaml,examples/eks-xr.yaml --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL)
@KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) $(UPTEST) e2e test/karpenter-xr.yaml,examples/network-xr.yaml,examples/eks-xr.yaml --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL)
@$(OK) running automated tests

# This target requires the following environment variables to be set:
Expand Down
51 changes: 20 additions & 31 deletions apis/composition.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ spec:
fromFieldPath: spec.parameters.region
toFieldPath: spec.forProvider.region
resources:
- name: InstanceNodeRole
- name: instanceNodeRole
base:
apiVersion: iam.aws.upbound.io/v1beta1
kind: Role
Expand Down Expand Up @@ -57,13 +57,19 @@ spec:
patchSetName: providerConfigRef
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.clusterName
toFieldPath: metadata.annotations[crossplane.io/external-name]
transforms:
- type: string
string:
fmt: "KarpenterNodeRole-%s"
- type: ToCompositeFieldPath
fromFieldPath: status.atProvider.arn
toFieldPath: status.karpenter.instanceProfileRoleArn
policy:
fromFieldPath: Optional
- type: ToCompositeFieldPath
fromFieldPath: status.roleArn
fromFieldPath: status.atProvider.arn
toFieldPath: status.karpenter.accountId
transforms:
- type: string
Expand All @@ -73,7 +79,7 @@ spec:
match: '::(\d+):'
group: 1

- name: InstanceNodeRoleEKSPolicy
- name: instanceNodeRoleEKSPolicy
base:
apiVersion: iam.aws.upbound.io/v1beta1
kind: RolePolicyAttachment
Expand Down Expand Up @@ -107,7 +113,7 @@ spec:
- type: PatchSet
patchSetName: deletionPolicy

- name: InstanceNodeRoleECRPolicy
- name: instanceNodeRoleECRPolicy
base:
apiVersion: iam.aws.upbound.io/v1beta1
kind: RolePolicyAttachment
Expand All @@ -124,7 +130,7 @@ spec:
- type: PatchSet
patchSetName: deletionPolicy

- name: InstanceNodeRoleSSMPolicy
- name: instanceNodeRoleSSMPolicy
base:
apiVersion: iam.aws.upbound.io/v1beta1
kind: RolePolicyAttachment
Expand All @@ -141,7 +147,7 @@ spec:
- type: PatchSet
patchSetName: deletionPolicy

- name: InstanceProfile
- name: instanceProfile
base:
apiVersion: iam.aws.upbound.io/v1beta1
kind: InstanceProfile
Expand Down Expand Up @@ -169,7 +175,7 @@ spec:

- name: IRSA
base:
apiVersion: aws.platformref.upbound.io/v1alpha1 # ToDo(haarchri): change apiVersion when split is done!
apiVersion: aws.platform.upbound.io/v1alpha1
kind: XIRSA
spec:
parameters:
Expand All @@ -196,6 +202,8 @@ spec:
policy:
fromFieldPath: Optional
- type: CombineFromComposite
policy:
fromFieldPath: Required
combine:
variables:
- fromFieldPath: spec.parameters.region
Expand Down Expand Up @@ -427,6 +435,8 @@ spec:
- type: PatchSet
patchSetName: region
- type: CombineFromComposite
policy:
fromFieldPath: Required
combine:
variables:
- fromFieldPath: status.karpenter.sqsQueueArn
Expand Down Expand Up @@ -750,6 +760,7 @@ spec:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
spec:
deletionPolicy: Orphan
forProvider:
namespace: karpenter
chart:
Expand All @@ -761,8 +772,6 @@ spec:
aws:
nodeNameConvention: ip-name
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- fromFieldPath: status.karpenter.IRSARoleArn
Expand Down Expand Up @@ -794,6 +803,7 @@ spec:
apiVersion: kubernetes.crossplane.io/v1alpha1
kind: Object
spec:
deletionPolicy: Orphan
forProvider:
manifest:
apiVersion: karpenter.sh/v1alpha5
Expand Down Expand Up @@ -828,8 +838,6 @@ spec:
providerRef:
name: default
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name

Expand All @@ -838,6 +846,7 @@ spec:
apiVersion: kubernetes.crossplane.io/v1alpha1
kind: Object
spec:
deletionPolicy: Orphan
forProvider:
manifest:
apiVersion: karpenter.k8s.aws/v1alpha1
Expand All @@ -850,8 +859,6 @@ spec:
NodeType: "default"
intent: apps
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- fromFieldPath: spec.parameters.id
Expand All @@ -864,21 +871,3 @@ spec:
toFieldPath: spec.forProvider.manifest.spec.instanceProfile
policy:
fromFieldPath: Required

# ToDo(haarchri): add usages for helm-chart and objects
# - name: XEKSUsagesChart
# base:
# apiVersion: apiextensions.crossplane.io/v1alpha1
# kind: Usage
# spec:
# of:
# apiVersion: aws.platform.upbound.io/v1alpha1
# kind: XEKS
# by:
# apiVersion: helm.crossplane.io/v1beta1
# kind: Release
# resourceSelector:
# matchControllerRef: true
# patches:
# - fromFieldPath: spec.parameters.providerConfigName
# toFieldPath: spec.of.resourceSelector.matchLabels[xeks.aws.platform.upbound.io/cluster-id]
2 changes: 0 additions & 2 deletions apis/definition.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ kind: CompositeResourceDefinition
metadata:
name: xkarpenters.aws.platform.upbound.io
spec:
connectionSecretKeys:
- kubeconfig
group: aws.platform.upbound.io
names:
kind: XKarpenter
Expand Down
4 changes: 1 addition & 3 deletions crossplane.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@ spec:
crossplane:
version: ">=v1.13.2-0"
dependsOn:
- configuration: xpkg.upbound.io/upbound/configuration-aws-eks
version: ">=v0.1.0"
- configuration: xpkg.upbound.io/upbound/configuration-aws-irsa
- configuration: xpkg.upbound.io/upbound/configuration-aws-eks-irsa
version: ">=v0.1.0"
- provider: xpkg.upbound.io/upbound/provider-aws-sqs
version: ">=v0.36.0"
Expand Down
9 changes: 5 additions & 4 deletions examples/eks-xr.yaml
Original file line number Diff line number Diff line change
@@ -1,19 +1,20 @@
apiVersion: aws.platform.upbound.io/v1alpha1
kind: XEKS
metadata:
name: configuration-aws-eks
name: configuration-aws-eks-karpenter
labels:
xeks.aws.platform.upbound.io/cluster-id: configuration-aws-eks-karpenter
spec:
parameters:
id: configuration-aws-eks
id: configuration-aws-eks-karpenter
region: us-west-2
version: "1.27"
iam:
# replace with your custom arn like:
# roleArn: arn:aws:iam::123456789:role/AWSReservedSSO_AdministratorAccess_d703c73ed340fde7
roleArn: ${data.aws_eks_iam_default_admin}
nodes:
count: 3
instanceType: t3.small
writeConnectionSecretToRef:
name: configuration-aws-eks-kubeconfig
name: configuration-aws-eks-karpenter-kubeconfig
namespace: upbound-system
5 changes: 3 additions & 2 deletions examples/karpenter-xr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ metadata:
name: configuration-aws-eks-karpenter
spec:
parameters:
clusterName: configuration-aws-eks-rktp2
id: configuration-aws-eks
# replace with your clusterName
clusterName:
id: configuration-aws-eks-karpenter
region: us-west-2
8 changes: 8 additions & 0 deletions examples/network-xr.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: aws.platform.upbound.io/v1alpha1
kind: XNetwork
metadata:
name: configuration-aws-eks-karpenter
spec:
parameters:
id: configuration-aws-eks-karpenter
region: us-west-2
8 changes: 8 additions & 0 deletions test/karpenter-xr.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: aws.platform.upbound.io/v1alpha1
kind: XKarpenter
metadata:
name: configuration-aws-eks-karpenter
spec:
parameters:
id: configuration-aws-eks-karpenter
region: us-west-2
35 changes: 35 additions & 0 deletions test/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,38 @@ spec:
namespace: upbound-system
source: Secret
EOF

SCRIPT_DIR=$( cd -- $( dirname -- "${BASH_SOURCE[0]}" ) &> /dev/null && pwd )

"${KUBECTL}" apply -f ${SCRIPT_DIR}/../examples/eks-xr.yaml

# Function to extract the annotation from a resource
get_annotation() {
local resource_json="$1"
local annotation="$2"
annotation_value=$(echo "$resource_json" | grep -o "\"$annotation\": \"[^\"]*\"" | cut -d '"' -f 4)
echo "$annotation_value"
}

# Watch for changes to the resource and extract the annotation
while true; do
resource_info=$(kubectl get cluster.eks.aws.upbound.io -o json)
annotation_value=$(get_annotation "$resource_info" "crossplane.io/external-name")

if [ -n "$annotation_value" ]; then
cat <<EOF | "${KUBECTL}" apply -f -
apiVersion: aws.platform.upbound.io/v1alpha1
kind: XKarpenter
metadata:
name: configuration-aws-eks-karpenter
spec:
parameters:
clusterName: $annotation_value
id: configuration-aws-eks-karpenter
region: us-west-2
EOF
exit 0
fi

sleep 1
done

0 comments on commit 37b4443

Please sign in to comment.