Enable make ci (#928)

.github/workflows/ci.yml

@@ -26,6 +26,9 @@ jobs:
     - name: Available platforms
       run: echo ${{ steps.buildx.outputs.platforms }}
 
+    - name: Run checks
+      run: make ci
+
     - name: Build
       env:
         REGISTRY: appscodeci

Dockerfile.test

@@ -0,0 +1,24 @@
+FROM debian:stretch
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+RUN set -x \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl bzip2
+
+RUN set -x                                                                                                                                                          \
+  && curl -fsSL -o restic.bz2 https://github.com/restic/restic/releases/download/v{RESTIC_VER}/restic_{RESTIC_VER}_{ARG_OS}_{ARG_ARCH}.bz2                          \
+  && bzip2 -d restic.bz2                                                                                                                                            \
+  && chmod 755 restic                                                                                                                                               \
+  && curl -fsSL -o restic_{NEW_RESTIC_VER}.bz2 https://github.com/restic/restic/releases/download/v{NEW_RESTIC_VER}/restic_{NEW_RESTIC_VER}_{ARG_OS}_{ARG_ARCH}.bz2 \
+  && bzip2 -d restic_{NEW_RESTIC_VER}.bz2                                                                                                                           \
+  && chmod 755 restic_{NEW_RESTIC_VER}
+
+
+
+FROM {ARG_FROM}
+
+COPY --from=0 restic /bin/restic
+COPY --from=0 restic_{NEW_RESTIC_VER} /bin/restic_{NEW_RESTIC_VER}
+COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/

Makefile

@@ -57,7 +57,7 @@ NEW_RESTIC_VER   := 0.9.5
 ###
 
 SRC_PKGS := apis client pkg
-SRC_DIRS := $(SRC_PKGS) *.go test hack/gendocs # directories which hold app source (not vendored)
+SRC_DIRS := $(SRC_PKGS) *.go test hack/gencrd hack/gendocs # directories which hold app source (not vendored)
 
 DOCKER_PLATFORMS := linux/amd64 linux/arm linux/arm64
 BIN_PLATFORMS    := $(DOCKER_PLATFORMS) windows/amd64 darwin/amd64
@@ -78,6 +78,7 @@ TAG_DBG          := $(VERSION)-dbg_$(OS)_$(ARCH)
 
 GO_VERSION       ?= 1.12.10
 BUILD_IMAGE      ?= appscode/golang-dev:$(GO_VERSION)-stretch
+TEST_IMAGE       ?= appscode/golang-dev:$(GO_VERSION)-stash
 
 OUTBIN = bin/$(OS)_$(ARCH)/$(BIN)
 ifeq ($(OS),windows)
@@ -88,12 +89,14 @@ endif
 BUILD_DIRS  := bin/$(OS)_$(ARCH)     \
                .go/bin/$(OS)_$(ARCH) \
                .go/cache             \
+               hack/config           \
                $(HOME)/.credentials  \
                $(HOME)/.kube         \
                $(HOME)/.minikube
 
 DOCKERFILE_PROD  = Dockerfile.in
 DOCKERFILE_DBG   = Dockerfile.dbg
+DOCKERFILE_TEST  = Dockerfile.test
 
 # If you want to build all binaries, see the 'all-build' rule.
 # If you want to build all containers, see the 'all-container' rule.
@@ -343,7 +346,21 @@ docker-manifest-%:
 .PHONY: test
 test: unit-tests e2e-tests
 
-unit-tests: $(BUILD_DIRS)
+bin/.container-$(DOTFILE_IMAGE)-TEST:
+	@echo "container: $(TEST_IMAGE)"
+	@sed                                            \
+	    -e 's|{ARG_BIN}|$(BIN)|g'                   \
+	    -e 's|{ARG_ARCH}|$(ARCH)|g'                 \
+	    -e 's|{ARG_OS}|$(OS)|g'                     \
+	    -e 's|{ARG_FROM}|$(BUILD_IMAGE)|g'          \
+	    -e 's|{RESTIC_VER}|$(RESTIC_VER)|g'         \
+	    -e 's|{NEW_RESTIC_VER}|$(NEW_RESTIC_VER)|g' \
+	    $(DOCKERFILE_TEST) > bin/.dockerfile-TEST-$(OS)_$(ARCH)
+	@DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --platform $(OS)/$(ARCH) --load --pull -t $(TEST_IMAGE) -f bin/.dockerfile-TEST-$(OS)_$(ARCH) .
+	@docker images -q $(TEST_IMAGE) > $@
+	@echo
+
+unit-tests: $(BUILD_DIRS) bin/.container-$(DOTFILE_IMAGE)-TEST
 	@docker run                                                 \
 	    -i                                                      \
 	    --rm                                                    \
@@ -355,7 +372,7 @@ unit-tests: $(BUILD_DIRS)
 	    -v $$(pwd)/.go/cache:/.cache                            \
 	    --env HTTP_PROXY=$(HTTP_PROXY)                          \
 	    --env HTTPS_PROXY=$(HTTPS_PROXY)                        \
-	    $(BUILD_IMAGE)                                          \
+	    $(TEST_IMAGE)                                           \
 	    /bin/bash -c "                                          \
 	        ARCH=$(ARCH)                                        \
 	        OS=$(OS)                                            \
@@ -464,13 +481,13 @@ verify-modules:
 	fi
 
 .PHONY: verify-gen
-verify-gen: gen
+verify-gen: gen fmt
 	@if !(git diff --quiet HEAD); then \
-		echo "generated files are out of date, run make gen"; exit 1; \
+		echo "files are out of date, run make gen fmt"; exit 1; \
 	fi
 
 .PHONY: ci
-ci: verify lint test build #cover
+ci: verify lint build unit-tests #cover
 
 .PHONY: qa
 qa:

README.md

@@ -1,5 +1,5 @@
 [![Go Report Card](https://goreportcard.com/badge/stash.appscode.dev/stash)](https://goreportcard.com/report/stash.appscode.dev/stash)
-[![Build Status](https://travis-ci.org/stashed/stash.svg?branch=master)](https://travis-ci.org/stashed/stash)
+[![Build Status](https://github.com/stashed/stash/workflows/CI/badge.svg)](https://github.com/stashed/stash/actions?workflow=CI)
 [![codecov](https://codecov.io/gh/stashed/stash/branch/master/graph/badge.svg)](https://codecov.io/gh/stashed/stash)
 [![Docker Pulls](https://img.shields.io/docker/pulls/appscode/stash.svg)](https://hub.docker.com/r/appscode/stash/)
 [![Slack](https://slack.appscode.com/badge.svg)](https://slack.appscode.com)

apis/stash/v1beta1/types_helpers.go

@@ -12,10 +12,7 @@ const (
 
 // TODO: complete
 func (t TargetRef) IsWorkload() bool {
-	if t.Kind == "Deployment" {
-		return true
-	}
-	return false
+	return t.Kind == "Deployment"
 }
 
 func GetOpenAPIDefinitionsWithRetentionPolicy(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {

apis/stash/v1beta1/validator.go

@@ -30,7 +30,7 @@ func (r RestoreSession) IsValid() error {
 	}
 
 	// ensure that no two rules with non-emtpy targetHosts matches for a host
-	res := make(map[string]int, 0)
+	res := make(map[string]int)
 	for i, rule := range r.Spec.Rules {
 		for _, host := range rule.TargetHosts {
 			v, ok := res[host]

apis/util.go

@@ -4,7 +4,8 @@ const (
 	StashKey   = "stash.appscode.com"
 	VersionTag = StashKey + "/tag"
 
-	KeyDeleteJobOnCompletion = StashKey + "/delete-job-on-completion"
+	KeyDeleteJobOnCompletion     = StashKey + "/delete-job-on-completion"
+	AllowDeletingJobOnCompletion = "true"
 )
 
 const (

go.mod

@@ -21,13 +21,14 @@ require (
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/robfig/cron/v3 v3.0.0
 	github.com/sirupsen/logrus v1.4.1 // indirect
+	github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect
 	github.com/spf13/afero v1.2.2
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.3
 	github.com/stretchr/testify v1.3.0
 	gomodules.xyz/cert v1.0.0
 	gomodules.xyz/envsubst v0.1.0
-	gomodules.xyz/stow v0.2.0
+	gomodules.xyz/stow v0.2.2
 	gopkg.in/ini.v1 v1.41.0
 	k8s.io/api v0.0.0-20190503110853-61630f889b3c
 	k8s.io/apiextensions-apiserver v0.0.0-20190516231611-bf6753f2aa24
@@ -40,7 +41,7 @@ require (
 	k8s.io/kube-aggregator v0.0.0-20190314000639-da8327669ac5
 	k8s.io/kube-openapi v0.0.0-20190502190224-411b2483e503
 	k8s.io/kubernetes v1.14.0
-	kmodules.xyz/client-go v0.0.0-20191016085444-3044854ca625
+	kmodules.xyz/client-go v0.0.0-20191021083014-0be9e7d5b970
 	kmodules.xyz/custom-resources v0.0.0-20190927035424-65fe358bb045
 	kmodules.xyz/objectstore-api v0.0.0-20191014210450-ac380fa650a3
 	kmodules.xyz/offshoot-api v0.0.0-20190901210649-de049192326c

go.sum

@@ -297,8 +297,9 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY=
+github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
@@ -415,8 +416,9 @@ gomodules.xyz/envsubst v0.1.0 h1:xvS0A4NeRwa8y3qW/+jgc13nT3rW1nlcuePDJujHzBM=
 gomodules.xyz/envsubst v0.1.0/go.mod h1:2o5f7bd13XIITbE2ZKieE05YkqB2KDoZkqKccGebduA=
 gomodules.xyz/jsonpatch/v2 v2.0.0 h1:OyHbl+7IOECpPKfVK42oFr6N7+Y2dR+Jsb/IiDV3hOo=
 gomodules.xyz/jsonpatch/v2 v2.0.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
-gomodules.xyz/stow v0.2.0 h1:GYymuy7sxAifaL0IXEmoHApeguXHK0hFT7ZQiYA8KTU=
 gomodules.xyz/stow v0.2.0/go.mod h1:YDDPM4nmJgQhxq4I5aGtLM/xrvJzDSVM/fbc7NGizpY=
+gomodules.xyz/stow v0.2.2 h1:t9TGRul+LZUdKHpnW4qZb1VkMylEaIXXGHMzXTB8uXk=
+gomodules.xyz/stow v0.2.2/go.mod h1:YDDPM4nmJgQhxq4I5aGtLM/xrvJzDSVM/fbc7NGizpY=
 gomodules.xyz/version v0.0.0-20190507203204-7cec7ee542d3/go.mod h1:Y8xuV02mL/45psyPKG3NCVOwvAOy6T5Kx0l3rCjKSjU=
 gomodules.xyz/version v0.1.0 h1:inGItCg/egI0jPMeIE0SQkiDIJaodOMoCrxYqasQLR0=
 gomodules.xyz/version v0.1.0/go.mod h1:Y8xuV02mL/45psyPKG3NCVOwvAOy6T5Kx0l3rCjKSjU=
@@ -487,8 +489,8 @@ k8s.io/kubernetes v1.14.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4=
 k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
 kmodules.xyz/client-go v0.0.0-20190808141354-bbb9e14f60ab/go.mod h1:1GI6h5D31op/2D+Hrn9DhXW6iUd5wsHBQRoLeY7fndM=
-kmodules.xyz/client-go v0.0.0-20191016085444-3044854ca625 h1:A/ceKV+HXdheKSpaY2CF7Q70E/iDKoysxiWtAuYMXyE=
-kmodules.xyz/client-go v0.0.0-20191016085444-3044854ca625/go.mod h1:wFUBDofZzgkLW1sFJT5SSt6/G3ph2P8zUi5v81q5BKQ=
+kmodules.xyz/client-go v0.0.0-20191021083014-0be9e7d5b970 h1:3XwVHSsPMMtTT4DWGRb97IyuuB4OtRSJf9KeaWTxdXQ=
+kmodules.xyz/client-go v0.0.0-20191021083014-0be9e7d5b970/go.mod h1:BChXGI3zd3lNSjUUfDcGmvufMdqrofLI/nrndcZ4LUo=
 kmodules.xyz/custom-resources v0.0.0-20190927035424-65fe358bb045 h1:DuvD64ouPDbv3egAUUCy5rBqs/vc218DeVVivcqfa+U=
 kmodules.xyz/custom-resources v0.0.0-20190927035424-65fe358bb045/go.mod h1:vlKyFcCXC+2Kfn3Fa5Z7RnBWyp4t46FSeEutNqpqMm8=
 kmodules.xyz/objectstore-api v0.0.0-20191014210450-ac380fa650a3 h1:64QSexLk/Dio4+L8Ge1tb4c44aBiwmUwTNP2kCu6YQU=

hack/gencrd/main.go

@@ -49,6 +49,7 @@ func generateSwaggerJson() {
 			stashv1beta1.GetOpenAPIDefinitions,
 			repov1alpha1.GetOpenAPIDefinitions,
 		},
+		//nolint:govet
 		Resources: []openapi.TypeInfo{
 			// v1alpha1 resources
 			{stashv1alpha1.SchemeGroupVersion, stashv1alpha1.ResourcePluralRestic, stashv1alpha1.ResourceKindRestic, true},
@@ -64,6 +65,7 @@ func generateSwaggerJson() {
 			{stashv1beta1.SchemeGroupVersion, stashv1beta1.ResourceKindFunction, stashv1beta1.ResourceKindFunction, false},
 			{stashv1beta1.SchemeGroupVersion, stashv1beta1.ResourcePluralTask, stashv1beta1.ResourceKindTask, false},
 		},
+		//nolint:govet
 		RDResources: []openapi.TypeInfo{
 			{repov1alpha1.SchemeGroupVersion, repov1alpha1.ResourcePluralSnapshot, repov1alpha1.ResourceKindSnapshot, true},
 		},

hack/gendocs/main.go

@@ -98,7 +98,10 @@ func main() {
 	linkHandler := func(name string) string {
 		return "/docs/reference/operator/" + name
 	}
-	doc.GenMarkdownTreeCustom(rootCmd, dir, filePrepender, linkHandler)
+	err = doc.GenMarkdownTreeCustom(rootCmd, dir, filePrepender, linkHandler)
+	if err != nil {
+		log.Fatalln(err)
+	}
 
 	index := filepath.Join(dir, "_index.md")
 	f, err := os.OpenFile(index, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)

hack/test.sh

@@ -26,27 +26,3 @@ TARGETS=$(for d in "$@"; do echo ./$d/...; done)
 echo "Running tests:"
 go test -installsuffix "static" ${TARGETS}
 echo
-
-echo -n "Checking gofmt: "
-ERRS=$(find "$@" -type f -name \*.go | xargs gofmt -l 2>&1 || true)
-if [ -n "${ERRS}" ]; then
-    echo "FAIL - the following files need to be gofmt'ed:"
-    for e in ${ERRS}; do
-        echo "    $e"
-    done
-    echo
-    exit 1
-fi
-echo "PASS"
-echo
-
-echo -n "Checking go vet: "
-ERRS=$(go vet ${TARGETS} 2>&1 || true)
-if [ -n "${ERRS}" ]; then
-    echo "FAIL"
-    echo "${ERRS}"
-    echo
-    exit 1
-fi
-echo "PASS"
-echo

pkg/backup/backup.go

@@ -221,7 +221,7 @@ func (c *Controller) setup() (*api.Restic, *api.Repository, error) {
 	if err := os.MkdirAll(c.opt.ScratchDir, 0755); err != nil {
 		return nil, nil, fmt.Errorf("failed to create scratch dir: %s", err)
 	}
-	if err := ioutil.WriteFile(c.opt.ScratchDir+"/.stash", []byte("test"), 644); err != nil {
+	if err := ioutil.WriteFile(c.opt.ScratchDir+"/.stash", []byte("test"), 0644); err != nil {
 		return nil, nil, fmt.Errorf("no write access in scratch dir: %s", err)
 	}
 
@@ -256,7 +256,7 @@ func (c *Controller) setup() (*api.Restic, *api.Repository, error) {
 }
 
 func (c *Controller) runResticBackup(restic *api.Restic, repository *api.Repository) (err error) {
-	if restic.Spec.Paused == true {
+	if restic.Spec.Paused {
 		log.Infoln("skipped logging since restic is paused.")
 		return nil
 	}
@@ -300,16 +300,21 @@ func (c *Controller) runResticBackup(restic *api.Restic, repository *api.Reposit
 			}
 			restic_session_duration_seconds_total.Set(endTime.Sub(startTime.Time).Seconds())
 
-			push.Collectors(c.JobName(restic),
-				c.GroupingKeys(restic),
-				c.opt.PushgatewayURL,
+			pusher := push.New(c.opt.PushgatewayURL, c.JobName(restic))
+			registry := prometheus.NewRegistry()
+			registry.MustRegister(
 				restic_session_success,
 				restic_session_fail,
 				restic_session_duration_seconds_total,
-				restic_session_duration_seconds)
+				restic_session_duration_seconds,
+			)
+			err := pusher.Gatherer(registry).Push()
+			if err != nil {
+				log.Errorln(err)
+			}
 		}
 		if err == nil {
-			stash_util.UpdateRepositoryStatus(c.stashClient.StashV1alpha1(), repository, func(in *api.RepositoryStatus) *api.RepositoryStatus {
+			_, err2 := stash_util.UpdateRepositoryStatus(c.stashClient.StashV1alpha1(), repository, func(in *api.RepositoryStatus) *api.RepositoryStatus {
 				in.BackupCount++
 				in.LastBackupTime = &startTime
 				if in.FirstBackupTime == nil {
@@ -318,6 +323,9 @@ func (c *Controller) runResticBackup(restic *api.Restic, repository *api.Reposit
 				in.LastBackupDuration = endTime.Sub(startTime.Time).String()
 				return in
 			})
+			if err2 != nil {
+				log.Errorln(err2)
+			}
 		}
 	}()
 
@@ -383,7 +391,7 @@ func (c *Controller) runResticBackup(restic *api.Restic, repository *api.Reposit
 func (c *Controller) measure(f func(*api.Restic, api.FileGroup) error, restic *api.Restic, fg api.FileGroup, g prometheus.Gauge) (err error) {
 	startTime := time.Now()
 	defer func() {
-		g.Set(time.Now().Sub(startTime).Seconds())
+		g.Set(time.Since(startTime).Seconds())
 	}()
 	err = f(restic, fg)
 	return

pkg/backup/backupsession.go

@@ -194,13 +194,13 @@ func (c *BackupSessionController) startBackupProcess(backupSession *api_v1beta1.
 	// step down from leadership so that another replica can acquire leadership and start taking backup.
 	switch backupConfiguration.Spec.Target.Ref.Kind {
 	case apis.KindDeployment, apis.KindReplicaSet, apis.KindReplicationController, apis.KindDeploymentConfig:
-		return c.backup(backupSession, backupConfiguration)
+		return c.backup(backupConfiguration)
 	default:
 		return nil, c.electBackupLeader(backupSession, backupConfiguration)
 	}
 }
 
-func (c *BackupSessionController) backup(backupSession *api_v1beta1.BackupSession, backupConfiguration *api_v1beta1.BackupConfiguration) (*restic.BackupOutput, error) {
+func (c *BackupSessionController) backup(backupConfiguration *api_v1beta1.BackupConfiguration) (*restic.BackupOutput, error) {
 
 	// get repository
 	repository, err := c.StashClient.StashV1alpha1().Repositories(backupConfiguration.Namespace).Get(backupConfiguration.Spec.Repository.Name, metav1.GetOptions{})
@@ -337,7 +337,7 @@ func (c *BackupSessionController) electBackupLeader(backupSession *api_v1beta1.B
 			OnStartedLeading: func(ctx context.Context) {
 				log.Infoln("Got leadership, preparing for backup")
 				// run backup process
-				backupOutput, backupErr := c.backup(backupSession, backupConfiguration)
+				backupOutput, backupErr := c.backup(backupConfiguration)
 				if backupErr != nil {
 					err := c.handleBackupFailure(backupSession.Name, backupErr)
 					if err != nil {