Merge pull request runatlantis#32 from hootsuite/e2e-tests

Adding ci for atlantis

Makefile

@@ -1,5 +1,6 @@
 BUILD_ID := $(shell git rev-parse --short HEAD 2>/dev/null || echo no-commit-id)
 WORKSPACE := $(shell pwd)
+PKG := $(shell go list ./... | grep -v e2e)
 
 .PHONY: test
 
@@ -14,21 +15,22 @@ debug: ## Output internal make variables
 	@echo BUILD_ID = $(BUILD_ID)
 	@echo IMAGE_NAME = $(IMAGE_NAME)
 	@echo WORKSPACE = $(WORKSPACE)
+	@echo PKG = $(PKG)
 
 deps:
-	go get .
+	go get -v $(PKG)
 
 build-service: ## Build the main Go service
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o atlantis .
 
 deps-test:
-	go get -t
+	go get -t $(PKG)
 
 test: ## Run tests, coverage reports, and clean (coverage taints the compiled code)
-	go test -v .
+	go test -v $(PKG)
 
 test-coverage:
-	go test -v -coverprofile=c.out
+	go test -v $(PKG) -coverprofile=c.out
 	go tool cover -html=c.out -o coverage.html
 
 dist: ## Package up everything in static/ using go-bindata-assetfs so it can be served by a single binary

README.md

@@ -1,4 +1,6 @@
 # atlantis
+[![CircleCI](https://circleci.com/gh/hootsuite/atlantis.svg?style=svg&circle-token=6a0c78c9b1fd77486c72a5e22512c7c9175f2aaf)](https://circleci.com/gh/hootsuite/atlantis)
+
 A [terraform](https://www.terraform.io/) collaboration tool that enables you to collaborate on infrastructure safely and securely.
 
 ## Locking

circle.yml

@@ -0,0 +1,41 @@
+machine:
+  # Environment variables for build
+  environment:
+    GOPATH:  "${HOME}/.go_workspace"
+    WORKDIR: "${GOPATH}/src/github.com/hootsuite/atlantis"
+    TERRAFORM_VERSION: 0.8.8
+
+dependencies:
+  pre:
+    # remove old go files
+    - rm -rf "$GOPATH"
+
+  override:
+    # move repository to the canonical import path
+    - mkdir -p "$(dirname ${WORKDIR})"
+    - cp -R "${HOME}/atlantis" "${WORKDIR}"
+    # Install dependencies
+    - cd "${WORKDIR}" && make deps
+
+test:
+  pre:
+    # Test dependencies
+    - cd "${WORKDIR}" && make deps-test
+
+  override:
+    # Run tests
+    - cd "${WORKDIR}" && ./scripts/build.sh
+
+  # Run e2e tests
+  post:
+    - cd "${WORKDIR}" && ./scripts/e2e-deps.sh
+    # Start atlantis server
+    - cd "${WORKDIR}/e2e" && ./atlantis server --gh-user="$GITHUB_USERNAME" --gh-password="$GITHUB_PASSWORD" --data-dir="/tmp" --require-approval=false --s3-bucket="$ATLANTIS_S3_BUCKET_NAME" --log-level="debug" &> /tmp/atlantis-server.log:
+          background: true
+    - sleep 2
+    - cd "${WORKDIR}/e2e" && ./ngrok http 4141:
+          background: true
+    - sleep 2
+    # Set ATLANTIS_URL environment variable to be used by atlantis e2e test to create the webhook
+    - echo '' >> ~/.circlerc && echo 'export ATLANTIS_URL=$(curl -s 'http://localhost:4040/api/tunnels' | jq -r '.tunnels[1].public_url') ' >> ~/.circlerc
+    - cd "${WORKDIR}/e2e" && ../scripts/e2e.sh

e2e/.gitconfig

@@ -0,0 +1,3 @@
+[user]
+	name = Anubhav Mishra
+	email = [email protected]

e2e/.gitignore

@@ -0,0 +1 @@
+atlantis-tests

e2e/Makefile

@@ -0,0 +1,25 @@
+WORKSPACE := $(shell pwd)
+
+.PHONY: test
+
+.DEFAULT_GOAL := help
+help: ## List targets & descriptions
+	@cat Makefile* | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+debug: ## Output internal make variables
+	@echo WORKSPACE = $(WORKSPACE)
+
+deps: ## Get go dependencies
+	go get -v .
+
+build: ## Build the main Go service
+	go build -v -o atlantis-tests .
+
+deps-test: ## Run tests for go dependencies
+	go get -t
+
+test: ## Run tests, coverage reports, and clean (coverage taints the compiled code)
+	go test -v .
+
+run: ## Run e2e tests
+	./atlantis-tests

e2e/e2e.go

@@ -0,0 +1,189 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/google/go-github/github"
+)
+
+type E2ETester struct {
+	githubClient *GithubClient
+	repoUrl      string
+	ownerName    string
+	repoName     string
+	hookID       int
+	cloneDirRoot string
+	projectType  Project
+}
+
+type E2EResult struct {
+	projectType          string
+	githubPullRequestURL string
+	testResult           string
+}
+
+var testFileData = `
+resource "null_resource" "hello" {
+}
+`
+
+func (t *E2ETester) Start() (*E2EResult, error) {
+	cloneDir := fmt.Sprintf("%s/%s-test", t.cloneDirRoot, t.projectType.Name)
+	branchName := fmt.Sprintf("%s-%s", t.projectType.Name, time.Now().Format("20060102150405"))
+	testFileName := fmt.Sprintf("%s.tf", t.projectType.Name)
+	e2eResult := &E2EResult{}
+	e2eResult.projectType = t.projectType.Name
+
+	// create the directory and parents if necessary
+	log.Printf("creating dir %q", cloneDir)
+	if err := os.MkdirAll(cloneDir, 0755); err != nil {
+		return e2eResult, fmt.Errorf("failed to create dir %q prior to cloning, attempting to continue: %v", cloneDir, err)
+	}
+
+	cloneCmd := exec.Command("git", "clone", t.repoUrl, cloneDir)
+	// git clone the repo
+	log.Printf("git cloning into %q", cloneDir)
+	if output, err := cloneCmd.CombinedOutput(); err != nil {
+		return e2eResult, fmt.Errorf("failed to clone repository: %v: %s", err, string(output))
+	}
+
+	// checkout a new branch for the project
+	log.Printf("checking out branch %q", branchName)
+	checkoutCmd := exec.Command("git", "checkout", "-b", branchName)
+	checkoutCmd.Dir = cloneDir
+	if err := checkoutCmd.Run(); err != nil {
+		return e2eResult, fmt.Errorf("failed to git checkout branch %q: %v", branchName, err)
+	}
+
+	// write a file for running the tests
+	randomData := []byte(testFileData)
+	filePath := fmt.Sprintf("%s/%s/%s", cloneDir, t.projectType.Name, testFileName)
+	log.Printf("creating file to commit %q", filePath)
+	err := ioutil.WriteFile(filePath, randomData, 0644)
+	if err != nil {
+		return e2eResult, fmt.Errorf("couldn't write file %s: %v", filePath, err)
+	}
+
+	// add the file
+	log.Printf("git add file %q", filePath)
+	addCmd := exec.Command("git", "add", filePath)
+	addCmd.Dir = cloneDir
+	if err := addCmd.Run(); err != nil {
+		return e2eResult, fmt.Errorf("failed to git add file %q: %v", filePath, err)
+	}
+
+	// commit the file
+	log.Printf("git commit file %q", filePath)
+	commitCmd := exec.Command("git", "commit", "-am", "test commit")
+	commitCmd.Dir = cloneDir
+	if output, err := commitCmd.CombinedOutput(); err != nil {
+		return e2eResult, fmt.Errorf("failed to run git commit in %q: %v", cloneDir, err, string(output))
+	}
+
+	// push the branch to remote
+	log.Printf("git push branch %q", branchName)
+	pushCmd := exec.Command("git", "push", "origin", branchName)
+	pushCmd.Dir = cloneDir
+	if err := pushCmd.Run(); err != nil {
+		return e2eResult, fmt.Errorf("failed to git push branch %q: %v", branchName, err)
+	}
+
+	// create a new pr
+	title := fmt.Sprintf("This is a test pull request for atlantis e2e test for %s project type", t.projectType.Name)
+	head := fmt.Sprintf("%s:%s", t.githubClient.username, branchName)
+	body := ""
+	base := "master"
+	newPullRequest := &github.NewPullRequest{Title: &title, Head: &head, Body: &body, Base: &base}
+
+	pull, _, err := t.githubClient.client.PullRequests.Create(t.githubClient.ctx, t.ownerName, t.repoName, newPullRequest)
+	if err != nil {
+		return e2eResult, fmt.Errorf("error while creating new pull request: %v", err)
+	}
+
+	// set pull request url
+	e2eResult.githubPullRequestURL = pull.GetHTMLURL()
+
+	log.Printf("created pull request %s", pull.GetHTMLURL())
+
+	// defer closing pull request and delete remote branch
+	defer cleanUp(t, pull.GetNumber(), branchName)
+
+	// create run plan comment
+	log.Printf("creating plan comment: %q", t.projectType.PlanCommand)
+	_, _, err = t.githubClient.client.Issues.CreateComment(t.githubClient.ctx, t.ownerName, t.repoName, pull.GetNumber(), &github.IssueComment{Body: github.String(t.projectType.PlanCommand)})
+	if err != nil {
+		return e2eResult, fmt.Errorf("error creating 'run plan' comment on github")
+	}
+
+	// wait for atlantis to respond to webhook
+	time.Sleep(2 * time.Second)
+
+	state := "not started"
+	// waiting for atlantis run and finish
+	for i := 0; i < 20 && checkStatus(state); i++ {
+		time.Sleep(2 * time.Second)
+		state, _ = getAtlantisStatus(t, branchName)
+		if state == "" {
+			log.Println("atlantis run hasn't started")
+			continue
+		}
+		log.Printf("atlantis run is in %s state", state)
+	}
+
+	log.Printf("atlantis run finished with %s status", state)
+	e2eResult.testResult = state
+	// check if atlantis run was a success
+	if state != "success" {
+		return e2eResult, fmt.Errorf("atlantis run project type %q failed with %s status", t.projectType.Name, state)
+	}
+
+	return e2eResult, nil
+}
+
+func getAtlantisStatus(t *E2ETester, branchName string) (string, error) {
+	// check repo status
+	combinedStatus, _, err := t.githubClient.client.Repositories.GetCombinedStatus(t.githubClient.ctx, t.ownerName, t.repoName, branchName, nil)
+	if err != nil {
+		return "", err
+	}
+
+	for _, status := range combinedStatus.Statuses {
+		if status.GetContext() == "Atlantis" {
+			return status.GetState(), nil
+		}
+	}
+
+	return "", nil
+}
+
+func checkStatus(state string) bool {
+	for _, s := range []string{"success", "error", "failure"} {
+		if state == s {
+			return false
+		}
+	}
+	return true
+}
+
+func cleanUp(t *E2ETester, pullRequestNumber int, branchName string) error {
+	// clean up
+	pullClosed, _, err := t.githubClient.client.PullRequests.Edit(t.githubClient.ctx, t.ownerName, t.repoName, pullRequestNumber, &github.PullRequest{State: github.String("closed")})
+	if err != nil {
+		return fmt.Errorf("error while closing new pull request: %v", err)
+	}
+	log.Printf("closed pull request %d", pullClosed.GetNumber())
+
+	deleteBranchName := fmt.Sprintf("%s/%s", "heads", branchName)
+	_, err = t.githubClient.client.Git.DeleteRef(t.githubClient.ctx, t.ownerName, t.repoName, deleteBranchName)
+	if err != nil {
+		return fmt.Errorf("error while deleting branch %s: %v", deleteBranchName, err)
+	}
+	log.Printf("deleted branch %s", deleteBranchName)
+
+	return nil
+}

e2e/github.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"context"
+
+	"github.com/google/go-github/github"
+)
+
+type GithubClient struct {
+	client   *github.Client
+	ctx      context.Context
+	username string
+}