Automate Pull Request Checklist - Check Tasks Completed

As described in #910, we
have a submitter checklists in pull requests that enforce with our
standards.

Currently, maintainers and reviewers have to manually check that the
submitter checklist has been checked off and verify each item in the
list. If the submitter checklist has not been checked off, they need
to follow up with the user and request them to follow the standards
and check them off.

In this change, we add automation that checks that the checklist has
been checked off. If there's an incomplete item, the check reports that
there are incomplete tasks but does not fail yet. In a future change, we
will make the check fail if the submitter checklist is not checked off.

tekton/ci-workspace/README.md

@@ -347,6 +347,7 @@ To react to pull requests:
         - ref: tekton-ci-webhook-pull-request
         - ref: tekton-ci-clone-depth
         - ref: tekton-ci-webhook-pr-labels
+        - ref: tekton-ci-webhook-pr-body
       template:
         ref: tekton-plumbing-ci-pipeline
 ```
@@ -388,6 +389,7 @@ To react to issue comments:
         - ref: tekton-ci-webhook-comment
         - ref: tekton-ci-clone-depth
         - ref: tekton-ci-webhook-pr-labels
+        - ref: tekton-ci-webhook-pr-body
       template:
         ref: tekton-plumbing-ci-pipeline
 ```

tekton/ci-workspace/catalog/trigger.yaml

@@ -23,6 +23,7 @@ spec:
     - ref: tekton-ci-webhook-pull-request
     - ref: tekton-ci-clone-depth
     - ref: tekton-ci-webhook-pr-labels
+    - ref: tekton-ci-webhook-pr-body
   template:
     ref: tekton-catalog-ci-pipeline
 ---

tekton/ci-workspace/community/trigger.yaml

@@ -23,6 +23,7 @@ spec:
     - ref: tekton-ci-webhook-pull-request
     - ref: tekton-ci-webhook-pr-labels
     - ref: tekton-ci-clone-depth
+    - ref: tekton-ci-webhook-pr-body
   template:
     ref: tekton-community-ci-pipeline
 ---

tekton/ci-workspace/plumbing/trigger.yaml

@@ -23,6 +23,7 @@ spec:
     - ref: tekton-ci-webhook-pull-request
     - ref: tekton-ci-webhook-pr-labels
     - ref: tekton-ci-clone-depth
+    - ref: tekton-ci-webhook-pr-body
   template:
     ref: tekton-plumbing-ci-pipeline
 ---

tekton/ci-workspace/shared/bindings.yaml

@@ -85,3 +85,13 @@ spec:
   params:
   - name: labels
     value: $(body.issue.labels)
+---
+apiVersion: triggers.tekton.dev/v1alpha1
+kind: TriggerBinding
+metadata:
+  name: tekton-ci-webhook-pr-body
+  namespace: tektonci
+spec:
+  params:
+    - name: labels
+      value: $(body.pull_request.body)

tekton/ci-workspace/shared/trigger.yaml

@@ -21,5 +21,6 @@ spec:
     - ref: tekton-ci-webhook-pull-request
     - ref: tekton-ci-webhook-pr-labels
     - ref: tekton-ci-clone-depth
+    - ref: tekton-ci-webhook-pr-body
   template:
     ref: tekton-doc-reviews

tekton/ci/README.md

@@ -334,6 +334,7 @@ To react to pull requests:
         - ref: tekton-ci-webhook-pull-request
         - ref: tekton-ci-clone-depth
         - ref: tekton-ci-webhook-pr-labels
+        - ref: tekton-ci-webhook-pr-body
       template:
         ref: tekton-plumbing-ci-pipeline
 ```
@@ -375,6 +376,7 @@ To react to issue comments:
         - ref: tekton-ci-webhook-comment
         - ref: tekton-ci-clone-depth
         - ref: tekton-ci-webhook-pr-labels
+        - ref: tekton-ci-webhook-pr-body
       template:
         ref: tekton-plumbing-ci-pipeline
 ```

tekton/ci/eventlistener.yaml

@@ -27,6 +27,7 @@ spec:
         - ref: tekton-ci-webhook-pull-request
         - ref: tekton-ci-clone-depth
         - ref: tekton-ci-webhook-pr-labels
+        - ref: tekton-ci-webhook-pr-body
       template:
         ref: tekton-plumbing-ci-pipeline
     - name: plumbing-comment-ci
@@ -120,6 +121,7 @@ spec:
         - ref: tekton-ci-github-base
         - ref: tekton-ci-webhook-pull-request
         - ref: tekton-ci-webhook-pr-labels
+        - ref: tekton-ci-webhook-pr-body
       template:
         ref: tekton-all-ci-pipeline
     - name: cd-trigger
@@ -176,6 +178,7 @@ spec:
         - ref: tekton-ci-webhook-pull-request
         - ref: tekton-ci-clone-depth
         - ref: tekton-ci-webhook-pr-labels
+        - ref: tekton-ci-webhook-pr-body
       template:
         ref: tekton-website-ci-pipeline
     - name: website-comment-ci
@@ -233,6 +236,7 @@ spec:
         - ref: tekton-ci-webhook-pull-request
         - ref: tekton-ci-clone-depth
         - ref: tekton-ci-webhook-pr-labels
+        - ref: tekton-ci-webhook-pr-body
       template:
         ref: tekton-community-ci-pipeline
     - name: community-comment-ci
@@ -290,6 +294,7 @@ spec:
         - ref: tekton-ci-webhook-pull-request
         - ref: tekton-ci-clone-depth
         - ref: tekton-ci-webhook-pr-labels
+        - ref: tekton-ci-webhook-pr-body
       template:
         ref: tekton-catalog-ci-pipeline
     - name: catalog-comment-ci

tekton/ci/jobs/kustomization.yaml

@@ -7,3 +7,4 @@ resources:
 - tekton-python-test.yaml
 - tekton-teps-validation.yaml
 - tekton-catlin-lint.yaml
+- tekton-github-tasks-completed.yaml

tekton/ci/jobs/tasks.yaml

@@ -132,3 +132,44 @@ spec:
         sys.exit(1)
       else:
         print("Exactly one \"kind/*\" label found: {}".format(validKindLabels))
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: github-tasks-completed
+  namespace: tektonci
+  description: |
+    Verifies that a PR has all the tasks completed
+spec:
+  params:
+    - name: body
+      description: The body of the Pull Request
+  stepTemplate:
+    env:
+      - name: HOME
+        value: /tekton/home
+  steps:
+    - name: install-pyyaml
+      image: python:3-alpine
+      script: |
+        pip install pyyaml --user
+    - name: check-github-tasks-completed
+      image: python:3-alpine
+      script: |
+        #!/usr/bin/env python
+
+        import sys
+
+        prBodyText = """$(params.body)"""
+
+        incompleteTasks = prBodyText.count("- []")
+
+        if incompleteTasks > 0:
+          msg = "Error: {} incomplete GitHub Tasks found, expecting no incomplete GitHub Tasks".format(incompleteTasks)
+          print(msg)
+          # TODO (jerop): comment out in a follow up PR
+          # Check failed. Return exit code 1.
+          # sys.exit(1)
+        else:
+          print("Found no incomplete GitHub Tasks)
+

tekton/ci/jobs/tekton-github-tasks-completed.yaml

@@ -0,0 +1,27 @@
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: tekton-github-tasks-completed
+  namespace: tektonci
+spec:
+  params:
+    - name: checkName
+      description: The name of the GitHub check that this pipeline is used for
+    - name: gitHubCommand
+      description: The command that was used to trigger testing
+    - name: body
+      description: The body of the Pull Request
+  tasks:
+    - name: github-tasks-completed
+      conditions:
+        - conditionRef: "check-name-matches"
+          params:
+            - name: gitHubCommand
+              value: $(params.gitHubCommand)
+            - name: checkName
+              value: $(params.checkName)
+      taskRef:
+        name: github-tasks-completed
+      params:
+        - name: body
+          value: $(params.body)

tekton/ci/templates/all-template.yaml

@@ -29,6 +29,8 @@ spec:
     default: ""
   - name: labels
     description: List of labels currently on the Pull Request
+  - name: body
+    description: The body of the Pull Request
   resourcetemplates:
   - apiVersion: tekton.dev/v1beta1
     kind: PipelineRun
@@ -51,4 +53,26 @@ spec:
         - name: checkName
           value: check-pr-has-kind-label
         - name: gitHubCommand
-          value: $(tt.params.gitHubCommand)
+          value: $(tt.params.gitHubCommand)
+  - apiVersion: tekton.dev/v1beta1
+    kind: PipelineRun
+    metadata:
+      generateName: check-github-tasks-completed-
+      labels:
+        prow.k8s.io/build-id: $(tt.params.buildUUID)
+        tekton.dev/check-name: check-pr-submitter-checklist
+        tekton.dev/kind: ci
+        tekton.dev/pr-number: $(tt.params.pullRequestNumber)
+      annotations:
+        tekton.dev/gitRevision: "$(tt.params.gitRevision)"
+        tekton.dev/gitURL: "$(tt.params.gitRepository)"
+    spec:
+      pipelineRef:
+        name: tekton-github-tasks-completed
+      params:
+        - name: body
+          value: $(tt.params.body)
+        - name: checkName
+          value: check-pr-submitter-checklist
+        - name: gitHubCommand
+          value: $(tt.params.gitHubCommand)

tekton/ci/templates/bindings.yaml

@@ -81,3 +81,13 @@ spec:
   params:
   - name: labels
     value: $(body.issue.labels)
+---
+apiVersion: triggers.tekton.dev/v1alpha1
+kind: TriggerBinding
+metadata:
+  name: tekton-ci-webhook-pr-body
+  namespace: tektonci
+spec:
+  params:
+    - name: labels
+      value: $(body.pull_request.body)