Generated v0.1.2 release

.github/workflows/presubmit.yaml

@@ -1,7 +1,9 @@
 name: Presubmit
 on:
   pull_request:
-    branches: [ master ]
+    branches: 
+    - main
+    - provisioner-work
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -11,7 +13,7 @@ jobs:
       with:
         go-version: 1.15.3
     - run: make toolchain
-    - run: echo ::add-path::/usr/local/kubebuilder/bin
+    - run: echo "/usr/local/kubebuilder/bin" >> $GITHUB_PATH
     - run: make ci
     - uses: actions/upload-artifact@v2
       with:

Makefile

@@ -1,20 +1,27 @@
 GOFLAGS ?= "-tags=${CLOUD_PROVIDER}"
+
+RELEASE_REPO ?= public.ecr.aws/b6u6q9h4
+RELEASE_VERSION ?= v0.1.2
+RELEASE_MANIFEST = releases/${CLOUD_PROVIDER}/manifest.yaml
+
 WITH_GOFLAGS = GOFLAGS=${GOFLAGS}
-RELEASE_VERSION ?= v0.1.0
+WITH_RELEASE_REPO = KO_DOCKER_REPO=${RELEASE_REPO}
 
 help: ## Display help
 	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-all: generate verify test ## Run all steps in the developer loop
+dev: codegen verify test ## Run all steps in the developer loop
 
-ci: generate verify battletest ## Run all steps used by continuous integration
+ci: codegen verify battletest ## Run all steps used by continuous integration
+
+release: publish helm docs ## Run all steps in release workflow
 
 test: ## Run tests
 	ginkgo -r
 
 battletest: ## Run stronger tests
 	# Ensure all files have cyclo-complexity =< 10
-	gocyclo -over 10 ./pkg
+	gocyclo -over 11 ./pkg
 	# Run randomized, parallelized, racing, code coveraged, tests
 	ginkgo -r \
 		-cover -coverprofile=coverage.out -outputdir=. -coverpkg=./pkg/... \
@@ -28,7 +35,7 @@ verify: ## Verify code. Includes dependencies, linting, formatting, etc
 	go fmt ./...
 	golangci-lint run
 
-generate: ## Generate code. Must be run if changes are made to ./pkg/apis/...
+codegen: ## Generate code. Must be run if changes are made to ./pkg/apis/...
 	controller-gen \
 		object:headerFile="hack/boilerplate.go.txt" \
 		webhook \
@@ -47,15 +54,21 @@ generate: ## Generate code. Must be run if changes are made to ./pkg/apis/...
 	perl -pi -e 's/Any/string/g' config/crd/bases/autoscaling.karpenter.sh_horizontalautoscalers.yaml
 	perl -pi -e 's/Any/string/g' config/crd/bases/autoscaling.karpenter.sh_scalablenodegroups.yaml
 	perl -pi -e 's/Any/string/g' config/crd/bases/autoscaling.karpenter.sh_metricsproducers.yaml
+	perl -pi -e 's/Any/string/g' config/crd/bases/provisioning.karpenter.sh_provisioners.yaml
 
 apply: ## Deploy the controller into your ~/.kube/config cluster
 	kubectl kustomize config | $(WITH_GOFLAGS) ko apply -B -f -
 
 delete: ## Delete the controller from your ~/.kube/config cluster
 	kubectl kustomize config | ko delete -f -
 
-release: ## Publish a versioned container image to $KO_DOCKER_REPO/karpenter and generate release manifests.
-	kubectl kustomize config | $(WITH_GOFLAGS) ko resolve -B -t $(RELEASE_VERSION) -f - > releases/${CLOUD_PROVIDER}/$(RELEASE_VERSION).yaml
+publish: ## Generate release manifests and publish a versioned container image.
+	kubectl kustomize config | $(WITH_RELEASE_REPO) $(WITH_GOFLAGS) ko resolve -B -t $(RELEASE_VERSION) -f - > $(RELEASE_MANIFEST)
+
+helm: ## Generate Helm Chart
+	cp $(RELEASE_MANIFEST) charts/karpenter/templates
+	yq w -i charts/karpenter/Chart.yaml version $(RELEASE_VERSION)
+	cd charts; helm package karpenter; helm repo index .
 
 docs: ## Generate Docs
 	gen-crd-api-reference-docs \
@@ -67,4 +80,4 @@ docs: ## Generate Docs
 toolchain: ## Install developer toolchain
 	./hack/toolchain.sh
 
-.PHONY: help all ci test release run apply delete verify generate docs toolchain
+.PHONY: help dev ci release test battletest verify codegen apply delete publish helm docs toolchain

NOTICE

@@ -0,0 +1,2 @@
+Karpenter
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

OWNERS

@@ -1,3 +1,7 @@
 approvers:
   - ellistarn
   - JacobGabrielson
+  - tabern
+reviewers:
+  - prateekgogia
+  - njtran

README.md

@@ -1,34 +1,53 @@
-# Karpenter
-![](./docs/images/logo.jpeg)
+![](./docs/images/karpenter-banner.png)
 
-Karpenter is a metrics-driven autoscaler for Kubernetes. It's performant, extensible, and can autoscale anything that implements the Kubernetes [scale subresource](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/autoscaling/horizontal-pod-autoscaler.md#scale-subresource).
+Karpenter is a metrics-driven autoscaler built for Kubernetes and can run in any Kubernetes cluster anywhere. It's performant, extensible, and can autoscale anything that implements the Kubernetes [scale subresource](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/autoscaling/horizontal-pod-autoscaler.md#scale-subresource).
+
+This is an early stage, experimental project built with ❤️ and is available as a **developer preview**. We're excited you are here - jump in, let us know what you think. We welcome contributions.
 
 ## Getting Started
-We will learn about Karpenter's APIs, look at some sample configurations, and install Karpenter's Controller.
+We will learn about Karpenter's APIs, look at some sample configurations, and install Karpenter's Controller. Alternatively, you can dive right into the [demo](https://github.com/ellistarn/karpenter-aws-demo).
 
 ### APIs
 Karpenter defines three custom resources to configure autoscaling behavior.
 
-**[HorizontalAutoscalers](./pkg/apis/autoscaling/v1alpha1/horizontalautoscaler.go)** define your autoscaling policy. It's modeled closely after the [HoriontalPodAutoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/), but has been generalized to support autoscaling for arbitrary resources. HorizontalAutoscalers periodically query metrics configured by `spec.metrics`, compute an autoscaling decision controlled by `spec.behavior`, and adjust the replicas of their `spec.scaleTargetRef`. Unlike the HPA, Karpenter's HorizontalAutoscalers integrate directly with Prometheus and can use any [promql](https://prometheus.io/docs/prometheus/latest/querying/basics/) response of type "instant vector" in their calculations.
+**[HorizontalAutoscalers](./pkg/apis/autoscaling/v1alpha1/horizontalautoscaler.go)** define your autoscaling policy. It's modeled closely after the [HorizontalPodAutoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/), but has been generalized to support autoscaling for arbitrary resources. HorizontalAutoscalers periodically query metrics configured by `spec.metrics`, compute an autoscaling decision controlled by `spec.behavior`, and adjust the replicas of their `spec.scaleTargetRef`. Unlike the HPA, Karpenter's HorizontalAutoscalers integrate directly with Prometheus and can use any [promql](https://prometheus.io/docs/prometheus/latest/querying/basics/) response of type "instant vector" in their calculations.
 
 **[MetricsProducers](./pkg/apis/autoscaling/v1alpha1/metricsproducer.go)** generate Prometheus metrics for commonly used autoscaling use cases. They periodically calculate a metric based on their configuration and expose it at a metrics endpoint that can be scraped by Prometheus. If you already have metrics you wish to use for autoscaling available in Prometheus, it is not necessary to define a Metrics Producer.
 
-**[ScalableNodeGroups](./pkg/apis/autoscaling/v1alpha1/scalablenodegroup.go)** provide a minimal way to point a HorizontalAutoscaler's `scaleTargetRef` to a Cloud Provider's Node Group API. Kubernetes core does not define an abstraction for Node Group. Instead, Cloud Providers typically expose non-Kubernetes Node Group APIs. ScalableNodeGroups are a shim in front of these APIs that are limited to `spec.replicas` and `status.replicas`. It is not a replcement or wrapper for these APIs. If you're using a solution that provides a Kubernetes API (e.g. [Kops](https://github.com/kubernetes/kops) or [Cluster API](https://github.com/kubernetes-sigs/cluster-api)), you can point the HorizontalAutoscaler's `scaleTargetRef` to these resources instead of a ScalableNodeGroup.
+**[ScalableNodeGroups](./pkg/apis/autoscaling/v1alpha1/scalablenodegroup.go)** provide a minimal way to point a HorizontalAutoscaler's `scaleTargetRef` to a Cloud Provider's Node Group API. Kubernetes core does not define an abstraction for Node Group. Instead, Cloud Providers typically expose non-Kubernetes Node Group APIs. ScalableNodeGroups are a shim in front of these APIs that are limited to `spec.replicas` and `status.replicas`. It is not a replacement or wrapper for these APIs. If you're using a solution that provides a Kubernetes API (e.g. [Kops](https://github.com/kubernetes/kops) or [Cluster API](https://github.com/kubernetes-sigs/cluster-api)), you can point the HorizontalAutoscaler's `scaleTargetRef` to these resources instead of a ScalableNodeGroup.
 
 [Learn more](./docs) about the different ways to configure Karpenter's resources.
 
 ## Installation
 Follow the setup recommendations of your cloud provider.
 - [AWS](./docs/aws/README.md#installation)
 
-Then install the controller.
+### Quick Install - Controller + Dependencies
 ```
-CLOUD_PROVIDER=aws
-VERSION=v0.1.0
-kubectl apply -f https://raw.githubusercontent.com/awslabs/karpenter/master/releases/${CLOUD_PROVIDER}/${VERSION}.yaml
+sh -c "$(curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/v0.1.2/hack/quick-install.sh)"
 ```
 
-# Docs
+### Kubectl - Standalone
+```
+kubectl apply -f https://raw.githubusercontent.com/awslabs/karpenter/v0.1.2/releases/aws/manifest.yaml
+```
+
+### Helm - Standalone
+```
+helm repo add karpenter https://awslabs.github.io/karpenter/charts
+helm install karpenter karpenter/karpenter
+```
+
+## Docs
 - [Examples](./docs/examples)
+- [Working Group](./docs/working-group)
 - [Developer Guide](./docs/DEVELOPER_GUIDE.md)
-- [Design](./docs/DESIGN.md)
+- [Design](./docs/designs/DESIGN.md)
+- [FAQs](./docs/FAQs.md)
+- [Contributing](./docs/CONTRIBUTING.md)
+
+## Terms
+Karpenter is an early stage, experimental project that is currently maintained by AWS and available as a preview. We request that you do not use Karpenter for production workloads at this time. See details in our [terms](./docs/TERMS.md).
+
+## License
+This project is licensed under the Apache-2.0 License.

charts/index.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+entries:
+  karpenter:
+  - apiVersion: v2
+    created: "2021-02-10T11:12:45.601847-08:00"
+    description: A Helm chart for https://github.com/awslabs/karpenter/.
+    digest: 3a64d3c51f5a706df905e49bdd7fb767d87658540864c4b97ffb959c3b15c8dd
+    name: karpenter
+    type: application
+    urls:
+    - karpenter-v0.1.2.tgz
+    version: v0.1.2
+  - apiVersion: v2
+    created: "2021-02-10T11:12:45.601157-08:00"
+    description: A Helm chart for https://github.com/awslabs/karpenter/.
+    digest: 39685c8cbe9a757ca48721aed08b49111fef18bc2a9f67d3223f19d0706f09f7
+    name: karpenter
+    type: application
+    urls:
+    - karpenter-v0.1.1.tgz
+    version: v0.1.1
+generated: "2021-02-10T11:12:45.599256-08:00"

charts/karpenter/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: cert-manager
+  repository: https://charts.jetstack.io
+  version: v1.1.0
+- name: kube-prometheus-stack
+  repository: https://prometheus-community.github.io/helm-charts
+  version: 12.3.0
+digest: sha256:5595919ac269b4105dd65d20eb27cb271b8976c1d10903e0b504d349df30f017
+generated: "2020-12-02T11:48:25.741819-08:00"

charts/karpenter/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: karpenter
+description: A Helm chart for https://github.com/awslabs/karpenter/.
+type: application
+version: v0.1.2

releases/aws/v0.1.0.yaml → charts/karpenter/templates/manifest.yaml

@@ -759,6 +759,15 @@ rules:
       - list
       - patch
       - watch
+  - apiGroups:
+      - '*'
+    resources:
+      - '*/scale'
+    verbs:
+      - update
+      - get
+      - list
+      - watch
   - apiGroups:
       - autoscaling.karpenter.sh
     resources:
@@ -913,7 +922,7 @@ spec:
         control-plane: karpenter
     spec:
       containers:
-        - image: 197575167141.dkr.ecr.us-west-2.amazonaws.com/karpenter:v0.1.0@sha256:b80ac089c17f15ac37c5f62780c9761e5725463f8a801cb4a4fb69af75c17949
+        - image: public.ecr.aws/b6u6q9h4/controller:v0.1.1@sha256:6a5c82cb34bbd6f714145cdfe7c14ac28404a00b56eec9b746ac61eeb3a6d6a8
           name: manager
           ports:
             - containerPort: 9443
@@ -940,7 +949,7 @@ spec:
             defaultMode: 420
             secretName: webhook-server-cert
 ---
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   labels:
@@ -956,7 +965,7 @@ spec:
     name: karpenter-selfsigned-issuer
   secretName: webhook-server-cert
 ---
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   labels:

karpenter/main.go → cmd/controller/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"github.com/awslabs/karpenter/pkg/controllers/provisioning/v1alpha1/reallocator"
 
 	"github.com/awslabs/karpenter/pkg/apis"
 	"github.com/awslabs/karpenter/pkg/cloudprovider"
@@ -12,15 +13,16 @@ import (
 	"github.com/awslabs/karpenter/pkg/autoscaler"
 	horizontalautoscalerv1alpha1 "github.com/awslabs/karpenter/pkg/controllers/horizontalautoscaler/v1alpha1"
 	metricsproducerv1alpha1 "github.com/awslabs/karpenter/pkg/controllers/metricsproducer/v1alpha1"
+	"github.com/awslabs/karpenter/pkg/controllers/provisioning/v1alpha1/allocator"
 	scalablenodegroupv1alpha1 "github.com/awslabs/karpenter/pkg/controllers/scalablenodegroup/v1alpha1"
 	metricsclients "github.com/awslabs/karpenter/pkg/metrics/clients"
 	"github.com/awslabs/karpenter/pkg/metrics/producers"
 
 	"github.com/awslabs/karpenter/pkg/utils/log"
-	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/runtime"
 
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	controllerruntimezap "sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -52,8 +54,7 @@ func main() {
 	flag.IntVar(&options.MetricsPort, "metrics-port", 8080, "The port the metric endpoint binds to for operating metrics about the controller itself.")
 	flag.Parse()
 
-	log.Setup(controllerruntimezap.UseDevMode(options.EnableVerboseLogging))
-
+	log.Setup(controllerruntimezap.UseDevMode(options.EnableVerboseLogging), controllerruntimezap.ConsoleEncoder())
 	manager := controllers.NewManagerOrDie(controllerruntime.GetConfigOrDie(), controllerruntime.Options{
 		LeaderElection:     true,
 		LeaderElectionID:   "karpenter-leader-election",
@@ -67,11 +68,15 @@ func main() {
 	metricsClientFactory := metricsclients.NewFactoryOrDie(options.PrometheusURI)
 	autoscalerFactory := autoscaler.NewFactoryOrDie(metricsClientFactory, manager.GetRESTMapper(), manager.GetConfig())
 
-	if err := manager.Register(
+	corev1Client, err := corev1.NewForConfig(manager.GetConfig())
+	log.PanicIfError(err, "Failed creating kube client")
+
+	err = manager.Register(
 		&horizontalautoscalerv1alpha1.Controller{AutoscalerFactory: autoscalerFactory},
 		&scalablenodegroupv1alpha1.Controller{CloudProvider: cloudProviderFactory},
 		&metricsproducerv1alpha1.Controller{ProducerFactory: metricsProducerFactory},
-	).Start(controllerruntime.SetupSignalHandler()); err != nil {
-		zap.S().Panicf("Unable to start manager, %w", err)
-	}
+		allocator.NewController(manager.GetClient(), corev1Client, cloudProviderFactory),
+		reallocator.NewController(manager.GetClient(), cloudProviderFactory),
+	).Start(controllerruntime.SetupSignalHandler())
+	log.PanicIfError(err, "Unable to start manager")
 }

config/crd/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - bases/autoscaling.karpenter.sh_horizontalautoscalers.yaml
 - bases/autoscaling.karpenter.sh_scalablenodegroups.yaml
 - bases/autoscaling.karpenter.sh_metricsproducers.yaml
+- bases/provisioning.karpenter.sh_provisioners.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:

config/manager/manager.yaml

@@ -22,9 +22,10 @@ spec:
       labels:
         control-plane: karpenter
     spec:
+      serviceAccountName: karpenter
       containers:
       - name: manager
-        image: ko://github.com/awslabs/karpenter/karpenter
+        image: ko://github.com/awslabs/karpenter/cmd/controller
         resources:
           limits:
             cpu: 100m

config/rbac/kustomization.yaml

@@ -1,4 +1,5 @@
 resources:
+- serviceaccount.yaml
 - role.yaml
 - role_binding.yaml
 - leader_election_role.yaml

config/rbac/leader_election_role_binding.yaml

@@ -8,5 +8,5 @@ roleRef:
   name: karpenter-leader-election
 subjects:
 - kind: ServiceAccount
-  name: default
+  name: karpenter
   namespace: karpenter