boskos: Deploying a new Boskos janitor for AWS

boskos/Makefile

@@ -32,6 +32,9 @@ reaper:
 janitor:
 	go build -o janitor/janitor k8s.io/test-infra/boskos/janitor/
 
+janitor-aws:
+	$(MAKE) -C ../maintenance/aws-janitor/cmd/aws-janitor-boskos
+
 metrics:
 	go build -o metrics/metrics k8s.io/test-infra/boskos/metrics/
 
@@ -53,6 +56,12 @@ janitor-image:
 	docker push "$(HUB)/janitor:$(TAG)"
 	rm janitor/janitor
 
+janitor-aws-image: export DOCKER_OPTS=--no-cache
+janitor-aws-image: export IMAGE=$(HUB)/janitor-aws
+janitor-aws-image:
+	$(MAKE) -C ../maintenance/aws-janitor/cmd/aws-janitor-boskos image push
+	$(MAKE) -C ../maintenance/aws-janitor/cmd/aws-janitor-boskos clean
+
 metrics-image:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o metrics/metrics k8s.io/test-infra/boskos/metrics/
 	docker build -t "$(HUB)/metrics:$(TAG)" metrics
@@ -81,4 +90,4 @@ update-config: get-cluster-credentials
 get-cluster-credentials:
 	gcloud container clusters get-credentials "$(CLUSTER)" --project="$(PROJECT)" --zone="$(ZONE)"
 
-.PHONY: boskos client reaper janitor metrics server-image reaper-image janitor-image metrics-image server-deployment reaper-deployment janitor-deployment metrics-deployment service update-config get-cluster-credentials
+.PHONY: boskos client reaper janitor janitor-aws metrics server-image reaper-image janitor-image janitor-aws-image metrics-image server-deployment reaper-deployment janitor-deployment metrics-deployment service update-config get-cluster-credentials

boskos/janitor/deployment.yaml

@@ -61,4 +61,24 @@ spec:
       - name: service
         secret:
           secretName: service-account
-
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: boskos-janitor-aws
+  labels:
+    app: boskos-janitor-aws
+  namespace: test-pods
+spec:
+  replicas: 4  # 4 distributed janitor instances
+  template:
+    metadata:
+      labels:
+        app: boskos-janitor-aws
+    spec:
+      terminationGracePeriodSeconds: 300
+      containers:
+      - name: boskos-janitor-aws
+        image: akutz/boskos-janitor-aws:v20190226-f06ce8bad
+        args:
+        - --boskos-url=http://boskos.test-pods.svc.cluster.local.

maintenance/aws-janitor/cmd/aws-janitor-boskos/.gitignore

@@ -0,0 +1 @@
+/aws-janitor-boskos

maintenance/aws-janitor/cmd/aws-janitor-boskos/Dockerfile

@@ -0,0 +1,25 @@
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM debian:stretch-20190204-slim
+
+RUN apt-get -y update && \
+    apt-get -y install ca-certificates && \
+    update-ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY aws-janitor-boskos /usr/local/bin/
+
+CMD ["aws-janitor-boskos"]
+ENTRYPOINT ["/bin/bash", "-c"]

maintenance/aws-janitor/cmd/aws-janitor-boskos/Makefile

@@ -0,0 +1,41 @@
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PROGRAM ?= aws-janitor-boskos
+HUB ?= gcr.io/k8s-testimages
+IMAGE ?= $(HUB)/$(notdir $(PROGRAM))
+TAG ?= $(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty)
+
+GOOS ?= $(shell go env GOOS)
+GOARCH ?= $(shell go env GOARCH)
+
+build: $(PROGRAM)
+$(PROGRAM):
+	CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o "$@"
+
+image:
+	GOOS=linux GOARCH=amd64 $(MAKE)
+	docker build $(DOCKER_OPTS) -t "$(IMAGE):$(TAG)" .
+
+push: image
+	docker push "$(IMAGE):$(TAG)"
+
+push-latest: push
+	docker tag "$(IMAGE):$(TAG)" "$(IMAGE):latest"
+	docker push "$(IMAGE):latest"
+
+clean:
+	rm -f $(PROGRAM)
+
+.PHONY: $(PROGRAM) build clean image push push-latest

maintenance/aws-janitor/cmd/aws-janitor-boskos/main.go

@@ -33,7 +33,10 @@ import (
 )
 
 var (
-	boskosURL = flag.String("boskos-url", "http://boskos", "Boskos URL")
+	boskosURL          = flag.String("boskos-url", "http://boskos", "Boskos URL")
+	sweepCount         = flag.Int("sweep-count", 3, "Number of times to sweep the resources")
+	sweepSleep         = flag.String("sweep-sleep", "30s", "The duration to pause between sweeps")
+	sweepSleepDuration time.Duration
 )
 
 const (
@@ -42,11 +45,18 @@ const (
 
 func main() {
 	flag.Parse()
+	if d, err := time.ParseDuration(*sweepSleep); err != nil {
+		sweepSleepDuration = time.Second * 30
+	} else {
+		sweepSleepDuration = d
+	}
+
 	logrus.SetFormatter(&logrus.JSONFormatter{})
 
 	boskos := client.NewClient("AWSJanitor", *boskosURL)
-	err := run(boskos)
-	logrus.WithError(err).Error("Janitor failure")
+	if err := run(boskos); err != nil {
+		logrus.WithError(err).Error("Janitor failure")
+	}
 }
 
 func run(boskos *client.Client) error {
@@ -83,8 +93,15 @@ func cleanResource(res *common.Resource) error {
 	logrus.WithField("name", res.Name).Info("beginning cleaning")
 	start := time.Now()
 
-	if err := resources.CleanAll(s, regions.Default); err != nil {
-		return errors.Wrapf(err, "Failed to clean resource %q", res.Name)
+	for i := 0; i < *sweepCount; i++ {
+		if err := resources.CleanAll(s, regions.Default); err != nil {
+			if i == *sweepCount-1 {
+				logrus.WithError(err).Warningf("Failed to clean resource %q", res.Name)
+			}
+		}
+		if i < *sweepCount-1 {
+			time.Sleep(sweepSleepDuration)
+		}
 	}
 
 	duration := time.Since(start)