Makefile and test script for ci-operator (#1)

* Makefile and test script for ci-operator

* Download kubectl, gcloud, ko binaries before tests

* Authenticate gcloud against the project

* Only set docker credentials helper

* Use full path to gcloud when authenticating

* Export USER env variable for serving tests

* Set a fake user as whoami doesnt work

* Set default kubeconfig only if user info available

* Ansible script for admission webhook and specific Docker image for ci-operator

Makefile

@@ -0,0 +1,29 @@
+#This makefile is used by ci-operator
+
+BUILD_DIR=$(shell pwd)/build
+GCLOUD_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-222.0.0-linux-x86_64.tar.gz
+GCLOUD_ARCHIVE=$(shell echo $(GCLOUD_URL) | rev | cut -d/ -f1 | rev)
+KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/v1.11.0/bin/linux/amd64/kubectl
+
+#TODO: Move this to a builder image in CI
+.PHONY: init
+init:
+	@echo "Downloading gcloud and authenticate"
+	@mkdir -p $(BUILD_DIR)/bin
+	@cd $(BUILD_DIR) && \
+	curl -LO $(GCLOUD_URL) && tar xzf $(GCLOUD_ARCHIVE) && \
+	google-cloud-sdk/bin/gcloud -q auth configure-docker
+	@echo "Downloading kubectl"
+	@cd $(BUILD_DIR)/bin && \
+	curl -LO $(KUBECTL_URL) && chmod +x ./kubectl
+	@echo "Downloading ko"
+	go get github.com/google/go-containerregistry/cmd/ko
+	@echo "Done preparing environment"
+
+.PHONY: clean
+clean:
+	rm -rf $(BUILD_DIR)
+
+.PHONY: test-e2e
+test-e2e:
+	sh openshift/e2e-tests-openshift.sh

openshift/admission-webhooks.yaml

@@ -0,0 +1,38 @@
+---
+- hosts: all
+  become: yes
+  become_user: root
+  tasks:
+    - name: Backup master-config.yaml to master-config.yaml.prepatch
+      copy:
+        src: /etc/origin/master/master-config.yaml
+        dest: /etc/origin/master/master-config.yaml.prepatch
+        remote_src: yes
+        backup: yes
+    - name: Set master_patch variable
+      set_fact:
+        master_patch: |
+          admissionConfig:
+            pluginConfig:
+              MutatingAdmissionWebhook:
+                configuration:
+                  apiVersion: apiserver.config.k8s.io/v1alpha1
+                  kubeConfigFile: /dev/null
+                  kind: WebhookAdmission
+              ValidatingAdmissionWebhook:
+                configuration:
+                  apiVersion: apiserver.config.k8s.io/v1alpha1
+                  kubeConfigFile: /dev/null
+                  kind: WebhookAdmission
+    - name: Apply patch for admission webhooks
+      shell: oc ex config patch /etc/origin/master/master-config.yaml.prepatch -p "{{ master_patch }}" > /etc/origin/master/master-config.yaml
+    - name: Restart API server and constollers
+      shell: /usr/local/bin/master-restart api && /usr/local/bin/master-restart controllers
+    - name: Wait for API server to be available
+      command: oc login -u system:admin
+      register: login_res
+      until: login_res.rc == 0
+      ignore_errors: yes
+      retries: 30
+      delay: 1
+

openshift/ci-operator/build-image/Dockerfile

@@ -0,0 +1,12 @@
+# Dockerfile to bootstrap build and test in openshift-ci
+
+FROM openshift/origin-release:golang-1.10
+
+# Add Google Cloud SDK repository
+ADD google-cloud-sdk.repo /etc/yum.repos.d/
+
+# Add kubernetes repository
+ADD kubernetes.repo /etc/yum.repos.d/
+
+RUN yum install -y google-cloud-sdk kubectl ansible && \
+    go get github.com/google/go-containerregistry/cmd/ko

openshift/ci-operator/build-image/google-cloud-sdk.repo

@@ -0,0 +1,9 @@
+[google-cloud-sdk]
+name=Google Cloud SDK
+baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
+       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+

openshift/ci-operator/build-image/kubernetes.repo

@@ -0,0 +1,7 @@
+[kubernetes]
+name=Kubernetes
+baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg

openshift/e2e-tests-openshift.sh

@@ -0,0 +1,124 @@
+#!/bin/sh 
+
+source $(dirname $0)/../test/cluster.sh
+
+export BUILD_DIR=`pwd`/../build
+export PATH=$BUILD_DIR/bin:$BUILD_DIR/google-cloud-sdk/bin:$PATH
+export K8S_CLUSTER_OVERRIDE=$(oc config current-context | awk -F'/' '{print $2}')
+export API_SERVER=$(oc config current-context | awk -F'/' '{print $2}' | awk -F':' '{print $1}')
+export DOCKER_REPO_OVERRIDE=gcr.io/$(gcloud config get-value project)/kserving-e2e-img
+export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE}
+export USER=testuser #satisfy e2e_flags.go#initializeFlags()
+
+env
+
+readonly ISTIO_URL='https://storage.googleapis.com/knative-releases/serving/latest/istio.yaml'
+readonly TEST_NAMESPACE=serving-tests
+
+function enable_admission_webhooks(){
+  echo "$API_SERVER ansible_ssh_private_key_file=~/.ssh/google_compute_engine" > inventory.ini
+  ansible-playbook ${REPO_ROOT_DIR}/openshift/admission-webhooks.yaml -i inventory.ini -u $KUBE_SSH_USER
+  rm inventory.ini
+}
+
+function install_istio(){
+  header "Installing Istio"
+  # Grant the necessary privileges to the service accounts Istio will use:
+  oc adm policy add-scc-to-user anyuid -z istio-ingress-service-account -n istio-system
+  oc adm policy add-scc-to-user anyuid -z default -n istio-system
+  oc adm policy add-scc-to-user anyuid -z prometheus -n istio-system
+  oc adm policy add-scc-to-user anyuid -z istio-egressgateway-service-account -n istio-system
+  oc adm policy add-scc-to-user anyuid -z istio-citadel-service-account -n istio-system
+  oc adm policy add-scc-to-user anyuid -z istio-ingressgateway-service-account -n istio-system
+  oc adm policy add-scc-to-user anyuid -z istio-cleanup-old-ca-service-account -n istio-system
+  oc adm policy add-scc-to-user anyuid -z istio-mixer-post-install-account -n istio-system
+  oc adm policy add-scc-to-user anyuid -z istio-mixer-service-account -n istio-system
+  oc adm policy add-scc-to-user anyuid -z istio-pilot-service-account -n istio-system
+  oc adm policy add-scc-to-user anyuid -z istio-sidecar-injector-service-account -n istio-system
+  oc adm policy add-cluster-role-to-user cluster-admin -z istio-galley-service-account -n istio-system
+
+  # Deploy the latest Istio release
+  oc apply -f $ISTIO_URL
+
+  # Ensure the istio-sidecar-injector pod runs as privileged
+  oc get cm istio-sidecar-injector -n istio-system -o yaml | sed -e 's/securityContext:/securityContext:\\n      privileged: true/' | oc replace -f -
+  # Monitor the Istio components until all the components are up and running
+  wait_until_pods_running istio-system || return 1
+  header "Istio Installed successfully"
+}
+
+function install_knative(){
+  header "Installing Knative"
+  # Grant the necessary privileges to the service accounts Knative will use:
+  oc adm policy add-scc-to-user anyuid -z build-controller -n knative-build
+  oc adm policy add-scc-to-user anyuid -z controller -n knative-serving
+  oc adm policy add-scc-to-user anyuid -z autoscaler -n knative-serving
+  oc adm policy add-cluster-role-to-user cluster-admin -z build-controller -n knative-build
+  oc adm policy add-cluster-role-to-user cluster-admin -z controller -n knative-serving
+
+  # Deploy Knative Serving from the current source repository. This will also install Knative Build.
+  create_serving
+
+  wait_until_pods_running knative-build || return 1
+  wait_until_pods_running knative-serving || return 1
+  wait_until_service_has_external_ip istio-system knative-ingressgateway || fail_test "Ingress has no external IP"
+  header "Knative Installed successfully"
+}
+
+function publish_test_images() {
+  header "Publishing test images"
+  image_dirs="$(find ${REPO_ROOT_DIR}/test/test_images -mindepth 1 -maxdepth 1 -type d)"
+  for image_dir in ${image_dirs}; do
+    ko publish -P "github.com/knative/serving/test/test_images/$(basename ${image_dir})"
+  done
+}
+
+function create_test_namespace(){
+  oc new-project $TEST_NAMESPACE
+}
+
+function run_e2e_tests(){
+  header "Running tests"
+  options=""
+  (( EMIT_METRICS )) && options="-emitmetrics"
+  report_go_test \
+    -v -tags=e2e -count=1 -timeout=20m \
+    ./test/conformance ./test/e2e \
+    --kubeconfig $KUBECONFIG \
+    ${options} || fail_test
+  success
+}
+
+function delete_istio(){
+  echo ">> Bringing down Istio"
+  oc delete --ignore-not-found=true -f ${ISTIO_URL}
+}
+
+function delete_test_namespace(){
+  echo ">> Deleting test namespace $TEST_NAMESPACE"
+  oc delete project $TEST_NAMESPACE
+}
+
+function teardown() {
+  delete_test_namespace
+  delete_test_resources
+  delete_serving
+  delete_istio
+}
+
+enable_admission_webhooks
+
+# Delete images in DOCKER_REPO_OVERRIDE repository and call teardown function
+teardown_test_resources
+
+create_test_namespace
+
+install_istio
+
+install_knative
+
+create_test_resources
+
+publish_test_images
+
+run_e2e_tests