CRD validation on flytepropeller project structure

.github/workflows/pull_request.yml

@@ -109,3 +109,12 @@ jobs:
           GO111MODULE: "on"
         with:
           args: make install && make lint
+
+  crd-check:
+    name: Check CRD code is up to date
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Check CRD code
+        run: DELTA_CHECK=true make op_code_generate

.gitignore

@@ -4,3 +4,4 @@ bin
 .DS_Store
 _test
 boilerplate/lyft/end2end/tmp
+github.com

Makefile

@@ -3,6 +3,24 @@ include boilerplate/flyte/docker_build/Makefile
 include boilerplate/flyte/golang_test_targets/Makefile
 include boilerplate/flyte/end2end/Makefile
 
+CRD_OPTIONS ?= "crd:generateEmbeddedObjectMeta=true"
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+OPENAPI_GEN = $(shell pwd)/bin/openapi-gen
+CRD_FILE =  $(shell pwd)/manifests/base/crds/flyteworkflow.flyte.net_flyteworkflows.yaml
+
+controller-gen: ## Download controller-gen locally if necessary.
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
+
+openapi-gen:
+	$(call go-get-tool,$(OPENAPI_GEN),k8s.io/kube-openapi/cmd/[email protected])
+
+manifests: controller-gen ## Generate CustomResourceDefinition objects.
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./pkg/apis/flyteworkflow/..." output:crd:artifacts:config=manifests/base/crds || true
+	# Kubernetes cannot allow additionalProperties value to be empty.
+	# More detail: https://github.com/kubernetes/kubernetes/issues/90504
+	perl -p -i -e 's/additionalProperties: {}/additionalProperties: true/g' $(CRD_FILE)
+	perl -p -i -e 's/flyteworkflows.flyteworkflow.flyte.net/flyteworkflows.flyte.lyft.com/g' $(CRD_FILE)
+	perl -p -i -e 's/flyteworkflow.flyte.net/flyte.lyft.com/g' $(CRD_FILE)
 
 .PHONY: update_boilerplate
 update_boilerplate:
@@ -26,7 +44,12 @@ cross_compile:
 	GOOS=linux GOARCH=amd64 go build -o bin/cross/flytepropeller ./cmd/controller/main.go
 	GOOS=linux GOARCH=amd64 go build -o bin/cross/kubectl-flyte ./cmd/kubectl-flyte/main.go
 
-op_code_generate:
+openapi_generate: openapi-gen
+	$(OPENAPI_GEN) -i github.com/flyteorg/flytepropeller/pkg/apis/flyteworkflow/v1alpha1 \
+				   -p github.com/flyteorg/flytepropeller/pkg/apis/flyteworkflow/v1alpha1 \
+				   --go-header-file hack/boilerplate.go.txt 2>&1 > /dev/null
+
+op_code_generate: openapi_generate manifests
 	@RESOURCE_NAME=flyteworkflow OPERATOR_PKG=github.com/flyteorg/flytepropeller ./hack/update-codegen.sh
 
 benchmark:
@@ -49,3 +72,17 @@ golden:
 .PHONY: generate
 generate: download_tooling
 	@go generate ./...
+
+# go-get-tool will 'go get' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(firstword $(MAKEFILE_LIST))))
+define go-get-tool
+@[ -f $(1) ] || { \
+set -e ;\
+TMP_DIR=$$(mktemp -d) ;\
+cd $$TMP_DIR ;\
+go mod init tmp ;\
+echo "Downloading $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
+rm -rf $$TMP_DIR ;\
+}
+endef

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/flyteorg/flyteplugins v0.7.1
 	github.com/flyteorg/flytestdlib v0.4.0
 	github.com/ghodss/yaml v1.0.0
+	github.com/go-openapi/spec v0.20.3
 	github.com/go-redis/redis v6.15.7+incompatible
 	github.com/go-test/deep v1.0.7
 	github.com/golang/protobuf v1.4.3
@@ -29,6 +30,7 @@ require (
 	k8s.io/apimachinery v0.20.2
 	k8s.io/client-go v0.20.2
 	k8s.io/klog v1.0.0
+	k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd
 	sigs.k8s.io/controller-runtime v0.8.2
 )
 

hack/update-codegen.sh

@@ -19,6 +19,7 @@
 set -o errexit
 set -o nounset
 set -o pipefail
+set +u
 
 : "${RESOURCE_NAME:?should be set for CRD}"
 : "${OPERATOR_PKG:?should be set for operator}"
@@ -28,16 +29,34 @@ echo "Generating CRD: ${RESOURCE_NAME}, in package ${OPERATOR_PKG}..."
 SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
 CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ../code-generator)}
 
-# generate the code with:
-# --output-base    because this script should also be able to run inside the vendor dir of
-#                  k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir
-#                  instead of the $GOPATH directly. For normal projects this can be dropped.
-bash ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \
-  ${OPERATOR_PKG}/pkg/client \
-  ${OPERATOR_PKG}/pkg/apis \
-  ${RESOURCE_NAME}:v1alpha1 \
-  --output-base "$(dirname ${BASH_SOURCE})/../../../.." \
-  --go-header-file ${SCRIPT_ROOT}/hack/boilerplate.go.txt
+if [[ -d ${CODEGEN_PKG} ]]; then
+    # generate the code with:
+    # --output-base    because this script should also be able to run inside the vendor dir of
+    #                  k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir
+    #                  instead of the $GOPATH directly. For normal projects this can be dropped.
+    bash ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \
+      ${OPERATOR_PKG}/pkg/client \
+      ${OPERATOR_PKG}/pkg/apis \
+      ${RESOURCE_NAME}:v1alpha1 \
+      --output-base "$(dirname ${BASH_SOURCE})/../../../.." \
+      --go-header-file ${SCRIPT_ROOT}/hack/boilerplate.go.txt
+fi
 
 # To use your own boilerplate text use:
 #   --go-header-file ${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt
+
+# This section is used by GitHub workflow to ensure that the generation step was run
+if [[ "$DELTA_CHECK" == true ]]; then
+  DIRTY=$(git status --porcelain)
+  if [ -n "$DIRTY" ]; then
+    echo "FAILED: CRD code updated without committing generated code."
+    echo "Ensure make op_code_generate has run and all changes are committed."
+    DIFF=$(git diff)
+    echo "diff detected: $DIFF"
+    DIFF=$(git diff --name-only)
+    echo "files different: $DIFF"
+    exit 1
+  else
+    echo "SUCCESS: Generated CRD code is up to date."
+  fi
+fi