Merge pull request #1 from jgunnink/terraform

Add Terraform Files

.gitignore

@@ -0,0 +1 @@
+terraform/.terraform/*

README.md

@@ -13,10 +13,16 @@ Currently, it has:
 
 ### Some notes
 
-- Function targets nodejs 14.
+- Function targets nodejs 16.
 - Function needs a personal access token from github to report back the status, with authorisation.
 
+### Using terraform
+
+1. The current only manual step is that inside a GCP project you'll need to create a secret with secret manager called GITHUB_TOKEN in order
+   to update github statuses.
+1. The personal access token from github needs "repo:status" only.
+
 ### On the todo list
 
-- Convert much of the infrastructure which has been defined via click ops to terraform.
-- Sort out a better way to report statuses back to github which don't involve a user's access token.
+[ ] Convert much of the infrastructure which has been defined via click ops to terraform.
+[ ] Sort out a better way to report statuses back to github which don't involve a user's access token.

functions/notify-github/cloudbuild.yaml

@@ -14,5 +14,5 @@ steps:
     args:
       - "-c"
       - |
-        gcloud functions deploy notify-github --runtime nodejs14 --trigger-http --region australia-southeast1 --entry-point notifyGithub
+        gcloud functions deploy notify-github --runtime nodejs16 --trigger-http --region us-central1 --entry-point notifyGithub
     dir: "functions/notify-github"

functions/notify-github/dist/index.js

@@ -8,6 +8,9 @@ var notifyGithub = function (req, res) {
     var repo = r.repo;
     var owner = r.owner || "jgunnink";
     var url = "https://api.github.com/repos/" + owner + "/" + repo + "/statuses/" + sha;
+    if (r.state === "failure") {
+        r.description = "Failed - " + r.description;
+    }
     var data = JSON.stringify({
         state: r.state,
         context: r.context,
@@ -26,10 +29,11 @@ var notifyGithub = function (req, res) {
     axios_1["default"]
         .post(url, data, config)
         .then(function (res) {
-        console.log("statusCode: " + res.status);
+        console.log("Github reponded with: " + res.status);
     })["catch"](function (error) {
         console.error(error);
     });
-    res.send("Notified Github: Pipeline Running");
+    res.send("Notified Github");
+    return data;
 };
 exports.notifyGithub = notifyGithub;

functions/notify-github/package.json

@@ -6,7 +6,7 @@
   "main": "dist/index.js",
   "scripts": {
     "start": "npx tsc-watch --onSuccess 'npx @google-cloud/functions-framework --target=notifyGithub'",
-    "deploy": "gcloud functions deploy notify-github --runtime nodejs14 --trigger-http --region australia-southeast1 --entry-point notifyGithub",
+    "deploy": "gcloud functions deploy notify-github --runtime nodejs16 --trigger-http --region us-central1 --entry-point notifyGithub",
     "build": "npx tsc",
     "test": "env TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\" }' mocha -r ts-node/register 'tests/**/*.ts'"
   },

functions/notify-github/src/index.ts

@@ -27,7 +27,7 @@ export const notifyGithub = (req: R, res: any) => {
     state: r.state,
     context: r.context,
     description: r.description,
-    target_url: `https://console.cloud.google.com/workflows/workflow/us-central1/workflow-1/execution/${r.workflowId}?project=${process.env.GCP_PROJECT}`,
+    target_url: `https://console.cloud.google.com/workflows/workflow/us-central1/${process.env.WORKFLOW_NAME}/execution/${r.workflowId}?project=${process.env.GCP_PROJECT}`,
   });
 
   const config = {

kickoff-workflow.yaml

@@ -4,4 +4,4 @@ steps:
     args:
       - "-c"
       - |
-        gcloud workflows execute workflow-1 --location=us-central1 --data={\"sha\":\"$COMMIT_SHA\"}
+        gcloud workflows execute ci-cd-pipeline --location=us-central1 --data={\"sha\":\"$COMMIT_SHA\"}

terraform-apply.yaml

@@ -0,0 +1,8 @@
+steps:
+  - name: "gcr.io/$PROJECT_ID/terraform"
+    args: ["init", "-reconfigure", "-backend-config=dev.tfbackend"]
+    dir: "./terraform"
+
+  - name: "gcr.io/$PROJECT_ID/terraform"
+    args: ["apply", "auto-approve", "-var-file=dev.tfvars"]
+    dir: "./terraform"

terraform-plan.yaml

@@ -0,0 +1,8 @@
+steps:
+  - name: 'gcr.io/$PROJECT_ID/terraform'
+    args: ["init", "-reconfigure", "-backend-config=dev.tfbackend"]
+    dir: "./terraform"
+
+  - name: "gcr.io/$PROJECT_ID/terraform"
+    args: ["plan", "-var-file=dev.tfvars"]
+    dir: "./terraform"

terraform/cloudbuild.tf

@@ -0,0 +1,68 @@
+resource "google_cloudbuild_trigger" "kickoff-workflow" {
+  name        = "kickoff-workflow"
+  description = "On pushes to the master branch, this trigger will activate, executing the workflow to run the pipeline"
+
+  github {
+    owner = var.repo_owner
+    name  = var.repo_name
+
+    push {
+      branch = "^master$"
+    }
+  }
+
+  filename = "kickoff-workflow.yaml"
+}
+
+resource "google_cloudbuild_trigger" "terraform-plan-ci" {
+  name        = "terraform-plan-ci"
+  description = "Run terraform plan on pull requests against master"
+
+  github {
+    owner = var.repo_owner
+    name  = var.repo_name
+
+    pull_request {
+      branch          = "^master$"
+      comment_control = "COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY"
+    }
+  }
+
+  filename = "terraform-plan.yaml"
+}
+
+resource "google_cloudbuild_trigger" "determine-differences" {
+  name        = "determine-differences"
+  description = "This cloudbuilder will compare two files or directories and based on the git hash given and the previous commit, will determine if there are changes with the compared files."
+
+  source_to_build {
+    uri       = "https://github.com/jgunnink/workflow-cicd-demo.git"
+    ref       = "refs/heads/main"
+    repo_type = "GITHUB"
+  }
+
+  git_file_source {
+    path      = "determineDifferences.yaml"
+    uri       = "https://github.com/jgunnink/workflow-cicd-demo.git"
+    revision  = "refs/heads/main"
+    repo_type = "GITHUB"
+  }
+}
+
+resource "google_cloudbuild_trigger" "deploy-infrastructure" {
+  name        = "deploy-infrastructure"
+  description = "This cloudbuilder run terraform apply."
+
+  source_to_build {
+    uri       = "https://github.com/jgunnink/workflow-cicd-demo.git"
+    ref       = "refs/heads/main"
+    repo_type = "GITHUB"
+  }
+
+  git_file_source {
+    path      = "terraform-apply.yaml"
+    uri       = "https://github.com/jgunnink/workflow-cicd-demo.git"
+    revision  = "refs/heads/main"
+    repo_type = "GITHUB"
+  }
+}

terraform/dev.tfbackend

@@ -0,0 +1 @@
+bucket = "terraform-state-bucket-cloud-pipeline-dev"

terraform/dev.tfvars

@@ -0,0 +1,5 @@
+region        = "us-central1"
+project_id    = "cloud-pipeline-dev"
+repo_name     = "workflow-cicd-demo"
+repo_owner    = "jgunnink"
+workflow_name = "ci-cd-pipeline"

terraform/functions.tf

@@ -0,0 +1,34 @@
+resource "google_cloudfunctions_function" "notify-github" {
+  available_memory_mb = "128"
+  entry_point         = "notifyGithub"
+
+  https_trigger_url = "https://${var.region}-${var.project_id}.cloudfunctions.net/notify-github"
+  ingress_settings  = "ALLOW_INTERNAL_ONLY"
+
+  labels = {
+    deployed-with = "terraform"
+    updated-with  = "cli-gcloud"
+  }
+
+  environment_variables = {
+    GCP_PROJECT   = var.project_id
+    WORKFLOW_NAME = var.workflow_name
+  }
+
+  secret_environment_variables {
+    key     = "GITHUB_TOKEN"
+    secret  = "GITHUB_TOKEN"
+    version = "1"
+  }
+
+  name                  = "notify-github"
+  project               = var.project_id
+  region                = var.region
+  runtime               = "nodejs16"
+  service_account_email = google_service_account.notify_github_sa.email
+  timeout               = 60
+  trigger_http          = true
+
+  source_archive_bucket = "gcf-sources-924919904854-australia-southeast1"
+  source_archive_object = "notify-github-fd1bee74-f30c-4aea-9972-ba5116e6da38/version-1/function-source.zip"
+}

terraform/iam.tf

@@ -0,0 +1,60 @@
+resource "google_project_iam_custom_role" "workflow_cloudbuild_invoker" {
+  description = "The custom role to enable invoking cloudbuild runs from the workflow service."
+  permissions = ["cloudbuild.builds.create", "cloudbuild.builds.get"]
+  project     = var.project_id
+  role_id     = "workflow_cloudbuild_invoker"
+  title       = "Workflow Cloudbuild Invoker"
+}
+
+resource "google_service_account" "workflow_runner_service_account" {
+  account_id   = "workflow-service"
+  description  = "Controls the workflow for the cloud pipeline"
+  display_name = "workflow-service"
+  project      = var.project_id
+}
+
+resource "google_service_account" "notify_github_sa" {
+  account_id   = "notify-github"
+  description  = "Controls the workflow for the cloud pipeline"
+  display_name = "notify-github"
+  project      = var.project_id
+}
+
+# google_project_iam_member: Non-authoritative.
+# Updates the IAM policy to grant a role to a new member. Other members for the role for the project are preserved.
+# https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam#google_project_iam_member
+resource "google_project_iam_member" "workflow_cloudbuild_invoker_custom" {
+  member  = "serviceAccount:${google_service_account.workflow_runner_service_account.email}"
+  project = var.project_id
+  role    = "projects/cloud-pipeline-dev/roles/workflow_cloudbuild_invoker"
+}
+
+resource "google_project_iam_member" "workflow_cloudfunction_invoker" {
+  member  = "serviceAccount:${google_service_account.workflow_runner_service_account.email}"
+  project = var.project_id
+  role    = "roles/cloudfunctions.invoker"
+}
+
+resource "google_project_iam_member" "notify_github_function_runner" {
+  member  = "serviceAccount:${google_service_account.notify_github_sa.email}"
+  project = var.project_id
+  role    = "roles/secretmanager.secretAccessor"
+}
+
+resource "google_project_iam_member" "cloudbuild_administer_cloudfunctions" {
+  member  = "serviceAccount:[email protected]"
+  project = var.project_id
+  role    = "roles/cloudfunctions.admin"
+}
+
+resource "google_project_iam_member" "cloudbuild_edit_workflows" {
+  member  = "serviceAccount:[email protected]"
+  project = var.project_id
+  role    = "roles/workflows.editor"
+}
+
+resource "google_project_iam_member" "cloudbuild_invoke_workflows" {
+  member  = "serviceAccount:[email protected]"
+  project = var.project_id
+  role    = "roles/workflows.invoker"
+}

terraform/main.tf

@@ -0,0 +1,15 @@
+terraform {
+  backend "gcs" {
+    # Bucket is passed in via cli arg. Eg, terraform init -reconfigure -backend-configuration=dev.tfbackend
+  }
+}
+
+provider "google" {
+  project = var.project_id
+  region  = var.region
+}
+
+provider "google-beta" {
+  project = var.project_id
+  region  = var.region
+}

terraform/variables.tf

@@ -0,0 +1,19 @@
+variable "project_id" {
+  description = "The project ID in Google Cloud to use for these resources."
+}
+
+variable "region" {
+  description = "The region in Google Cloud where the resources will be deployed."
+}
+
+variable "repo_name" {
+  description = "The name of the respository for cloudbuild to use with triggers"
+}
+
+variable "repo_owner" {
+  description = "The name of the owner of the repo. For example, this could be GoogleCloudPlatform or jgunnink"
+}
+
+variable "workflow_name" {
+  description = "The name of the workflow which will run the pipeline"
+}

terraform/workflow.tf

@@ -0,0 +1,7 @@
+resource "google_workflows_workflow" "workflow" {
+  name            = var.workflow_name
+  region          = "asia-southeast1"
+  description     = "CI/CD Pipeline"
+  service_account = "projects/${var.project_id}/serviceAccounts/${google_service_account.workflow_runner_service_account.email}"
+  source_contents = file("../workflow.yaml")
+}

workflow.yaml

@@ -10,7 +10,7 @@ main:
     - pendingGithub:
         call: http.post
         args:
-          url: https://australia-southeast1-cloud-pipeline-dev.cloudfunctions.net/notify-github
+          url: https://us-central1-cloud-pipeline-dev.cloudfunctions.net/notify-github
           auth:
             type: OIDC
           body:
@@ -99,7 +99,7 @@ main:
     - succeedGithub:
         call: http.post
         args:
-          url: https://australia-southeast1-cloud-pipeline-dev.cloudfunctions.net/notify-github
+          url: https://us-central1-cloud-pipeline-dev.cloudfunctions.net/notify-github
           auth:
             type: OIDC
           body:
@@ -115,7 +115,7 @@ main:
     - failGithub:
         call: http.post
         args:
-          url: https://australia-southeast1-cloud-pipeline-dev.cloudfunctions.net/notify-github
+          url: https://us-central1-cloud-pipeline-dev.cloudfunctions.net/notify-github
           auth:
             type: OIDC
           body: