Kubeflow README

README.md

@@ -2,7 +2,7 @@
 
 ![Swiss-army-kube](https://github.com/provectus/swiss-army-kube/raw/master/logo-swiss-army.png)
 
-## Prerequsite
+## Prerequisites
 
 #### Helm v3  
 `brew install helm`
@@ -20,19 +20,19 @@
 `brew install terraform`
 
 #### kfctl 
-- Downloads bin from https://github.com/kubeflow/kfctl/releases/  
-- Untar and open kfctl  
-( To run kfctl, go to the kfctl binary file in Finder, right-click, then select Open. Then click Open again to confirm that you want to open the app. )  
+`bash swiss-army-kube/kfctl_install.sh`
 
-- Move kfctl to bin and try get version
-```
-mv ~/Downloads/kfctl /usr/local/bin/kfctl
-kfctl version
-```
+( To run kfctl, go to the `/usr/local/bin/kfctl` binary file in Finder, right-click, then select Open. Then click Open again to confirm that you want to open the app. )  
+
+#### jq
+`brew install jq`
+
+#### To install all prerequisites
+`bash swiss-army-kube/prerequisites_install.sh` 
 
 ## Usage
 - Checkout repo
-- `cd swiss-army-kube/example` or rename "example" for your environment name and `cd <environment name>`
+- `cd swiss-army-kube/example` or rename "example" for your environment name and `cd swiss-army-kube/<environment name>`
 - `mv example.tfvars terraform.tfvars`
 - Edit terraform.tfvars (for security reasons terraform.tfvars in .gitignore)
 - Run `terraform init`

example/README.md

@@ -1,24 +1,30 @@
-# Prerequsite
+# Prerequisites
 
-Helm v3 - `brew install helm`
+#### Helm v3  
+`brew install helm`
 
-kubectl - `brew install kubernetes-cli`
+#### kubectl  
+`brew install kubernetes-cli`
 
-awscli - `brew install awscli`
+#### awscli  
+`brew install awscli`
 
-aws-iam-authenticator - `brew install aws-iam-authenticator`
+#### aws-iam-authenticator  
+`brew install aws-iam-authenticator`
 
-terraform - `brew install terraform`
+#### terraform  
+`brew install terraform`
 
-### Kubeflow
-kfctl - 
-Downloads bin from https://github.com/kubeflow/kfctl/releases/
-Untar and open kfctl ( To run kfctl, go to the kfctl binary file in Finder, right-click, then select Open. Then click Open again to confirm that you want to open the app. )
-Move kfctl to bin and try get version
-```
-mv ~/Downloads/kfctl /usr/local/bin/kfctl
-kfctl version
-```
+#### kfctl 
+`bash swiss-army-kube/kfctl_install.sh`
+
+( To run kfctl, go to the `/usr/local/bin/kfctl` binary file in Finder, right-click, then select Open. Then click Open again to confirm that you want to open the app. )  
+
+#### jq
+`brew install jq`
+
+#### To install all prerequisites
+`bash swiss-army-kube/prerequisites_install.sh` 
 
 # Structure
   main.tf - data from modules
@@ -96,4 +102,4 @@ Recreate resources
 
 If `terraform destroy` command fails, run
 `destroy_fix.sh`
-and try `terraform destroy` again
+and try `terraform destroy` again. After successful destroy process go to AWS console and delete argo-artifacts S3 bucket (if needed), also delete Route53 resources remaining from your deployment.

example/destroy.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+terraform destroy -auto-approve
+RESULT=$?
+
+if [ $RESULT -ne 0 ]; then
+    terraform state list | grep -e "module.*helm" | xargs terraform state rm
+    terraform state list | grep -e "module.*route53" | xargs terraform state rm
+
+    terraform state rm module.nginx.kubernetes_namespace.ingress-system
+    terraform state rm module.system.kubernetes_namespace.cert-manager
+
+    terraform state rm module.argo-artifacts.aws_s3_bucket.argo-artifacts
+
+    case "$OSTYPE" in
+      darwin*)  sed -i "" 's/\"skip_final_snapshot\":.*/\"skip_final_snapshot\": true,/g' "$SCRIPTPATH/terraform.tfstate" ;;
+      linux*)   sed -i 's/\"skip_final_snapshot\":.*/\"skip_final_snapshot\": true,/g' "$SCRIPTPATH/terraform.tfstate" ;;
+    esac
+
+    terraform destroy -auto-approve
+fi

example/destroy_fix.sh

@@ -1,7 +1,19 @@
 #!/bin/bash
 
+SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+# Workaround for failure of removing Route53 hosted zones
 terraform state list | grep -e "module.*helm" | xargs terraform state rm
 terraform state list | grep -e "module.*route53" | xargs terraform state rm
 
 terraform state rm module.nginx.kubernetes_namespace.ingress-system
-terraform state rm module.system.kubernetes_namespace.cert-manager
+terraform state rm module.system.kubernetes_namespace.cert-manager
+
+# Workaround for non-empty s3 bucket
+terraform state rm module.argo-artifacts.aws_s3_bucket.argo-artifacts
+
+# Workaround for "Error: RDS Cluster FinalSnapshotIdentifier is required when a final snapshot is required"
+case "$OSTYPE" in
+  darwin*)  sed -i "" 's/\"skip_final_snapshot\":.*/\"skip_final_snapshot\": true,/g' "$SCRIPTPATH/terraform.tfstate" ;;
+  linux*)   sed -i 's/\"skip_final_snapshot\":.*/\"skip_final_snapshot\": true,/g' "$SCRIPTPATH/terraform.tfstate" ;;
+esac

example/providers.tf

@@ -38,3 +38,7 @@ provider "null" {
 provider "template" {
   version = "~> 2.1"
 }
+
+provider "external" {
+  version = "~> 1.2"
+}

kfctl_install.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+case "$OSTYPE" in
+  darwin*)  OS="darwin" ;;
+  linux*)   OS="linux" ;;
+esac
+
+cd /tmp
+
+curl -s https://api.github.com/repos/kubeflow/kfctl/releases/latest \
+| grep "browser_download_url.*kfctl.*$OS.*\.tar\.gz" \
+| cut -d ":" -f 2,3 \
+| tr -d \" \
+| wget -qi -
+
+tarball="$(find . -name "*kfctl*$OS*.tar.gz" 2>/dev/null)"
+tar -xzf $tarball
+
+chmod +x kfctl
+mv kfctl /usr/local/bin/
+
+rm -f $tarball
+cd - &>/dev/null
+
+echo "Installed kfctl version:"
+kfctl version

modules/kubeflow/README.md

@@ -4,18 +4,67 @@ The Kubeflow project is dedicated to making deployments of machine learning (ML)
 
 
 ## Requirements
+The Kubernetes cluster must meet the following minimum requirements:
 
+* Your cluster must include at least one worker node with a minimum of:
+  * 4 CPU
+  * 50 GB storage
+  * 12 GB memory
+* The recommended Kubernetes version is 1.14. Kubeflow has been validated and tested on Kubernetes 1.14.
+  * Your cluster must run at least Kubernetes version 1.11.
+  * Kubeflow does not work on Kubernetes 1.16.
+  * Older versions of Kubernetes may not be compatible with the latest Kubeflow versions. The following matrix provides information about compatibility between Kubeflow and Kubernetes versions.
 
-## Settings
+| Kubernetes Versions | Kubeflow 0.4 | Kubeflow 0.5 | Kubeflow 0.6 | Kubeflow 0.7 | Kubeflow 1.0    |
+|---------------------|--------------|--------------|--------------|--------------|-----------------|
+| 1.11                | compatible   | compatible   | incompatible | incompatible | incompatible    |
+| 1.12                | compatible   | compatible   | incompatible | incompatible | incompatible    |
+| 1.13                | compatible   | compatible   | incompatible | incompatible | incompatible    |
+| 1.14                | compatible   | compatible   | compatible   | compatible   | compatible      |
+| 1.15                | incompatible | compatible   | compatible   | compatible   | compatible      |
+| 1.16                | incompatible | incompatible | incompatible | incompatible | no known issues |
+| 1.17                | incompatible | incompatible | incompatible | incompatible | no known issues |
+| 1.18                | incompatible | incompatible | incompatible | incompatible | no known issues |
 
+* incompatible: the combination does not work at all
+* compatible: all Kubeflow features have been tested and verified for the Kubernetes version
+* no known issues: the combination has not been fully tested but there are no repoted issues
 
-## How update SAK module kubeflow
+## Prerequisites
 
+#### kubectl  
+`brew install kubernetes-cli`
+
+#### awscli  
+`brew install awscli`
+
+#### aws-iam-authenticator  
+`brew install aws-iam-authenticator`
+
+#### kfctl 
+`bash swiss-army-kube/kfctl_install.sh`
+
+( To run kfctl, go to the `/usr/local/bin/kfctl` binary file in Finder, right-click, then select Open. Then click Open again to confirm that you want to open the app. )
+
+#### jq
+`brew install jq`
+
+## How to update SAK module kubeflow
+To add some modifications or custom overlays make changes in `modules/kubeflow/sak_kustomize` folder and `kfctl.yaml` configuration file. `modules/kubeflow/sak_kustomize` folder contains delta which is applied to original Kubeflow modules during deployment. This folder has same structure as original Kubeflow `kustomize` folder.
+
+__NOTE!!!__ any changes in `modules/kubeflow/sak_kustomize` folder and `kfctl.yaml` configuration file will trigger Kubeflow terraform resources recreation.
+
+To apply changes run:
+```
+terraform plan -out plan
+terraform apply plan
+```
 
 ## Dashboard access
 
 ```
 export NAMESPACE=istio-system
-KUBECONFIG=kubeconfig_swiss-test kubectl port-forward -n ${NAMESPACE} svc/istio-ingressgateway 8080:80
+KUBECONFIG=kubeconfig_swiss-army kubectl port-forward -n ${NAMESPACE} svc/istio-ingressgateway 8080:80
 ```
+
 Then open browser http://127.0.0.1:8080

modules/kubeflow/dir_md5sum.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# This script calculates the MD5 checksum on a directory
+#
+
+# Exit if any of the intermediate steps fail
+set -e
+
+# Extract "DIRECTORY" argument from the input into
+# DIRECTORY shell variables.
+# jq will ensure that the values are properly quoted
+# and escaped for consumption by the shell.
+eval "$(jq -r '@sh "DIRECTORY=\(.directory)"')"
+
+# Placeholder for whatever data-fetching logic your script implements
+CHECKSUM=`find ${DIRECTORY} -type f | LC_ALL=C sort | xargs shasum -a 256 | awk '{ n=split ($2, tokens, /\//); print $1 " " tokens[n]} ' |  shasum -a 256 | awk '{ print $1 }'`
+
+# Safely produce a JSON object containing the result value.
+# jq will ensure that the value is properly quoted
+# and escaped to produce a valid JSON string.
+jq -n --arg checksum "$CHECKSUM" '{"checksum":$checksum}'

modules/kubeflow/kustomize.tf

@@ -1,3 +1,11 @@
+data "external" "trigger" {
+  program = ["bash", "${path.module}/dir_md5sum.sh"]
+
+  query = {
+    directory = "${path.module}/sak_kustomize"
+  }
+}
+
 resource "local_file" "api-service" {
   depends_on = [
     null_resource.sak_kustomize
@@ -12,6 +20,7 @@ STORAGE_HOST=s3.${data.aws_region.this.id}.amazonaws.com
 STORAGE_BUCKET=${var.artifacts.id}
 STORAGE_KEY=${aws_iam_access_key.this.id}
 STORAGE_SECRET=${aws_iam_access_key.this.secret}
+# ${null_resource.sak_kustomize.id}
 EOT
   filename = "${path.module}/kustomize/api-service/base/params.env"
 }
@@ -26,6 +35,7 @@ MYSQL_ALLOW_EMPTY_PASSWORD=true
 MYSQL_DATABASE=${aws_rds_cluster.db.database_name}
 MYSQL_HOST=${aws_rds_cluster.db.endpoint}
 MYSQL_PORT=3306
+# ${null_resource.sak_kustomize.id}
 EOT
   filename = "${path.module}/kustomize/metadata/overlays/db/params.env"
 }
@@ -37,21 +47,30 @@ resource "local_file" "metadata-secrets" {
   content  = <<-EOT
 MYSQL_ROOT_PASSWORD=${aws_rds_cluster.db.master_password}
 MYSQL_USER_NAME=${aws_rds_cluster.db.master_username}
+# ${null_resource.sak_kustomize.id}
 EOT
   filename = "${path.module}/kustomize/metadata/overlays/db/secrets.env"
 }
 
 resource "null_resource" "kfctl_build" {
+  triggers = {
+    kfctl = "${sha1(file("${path.module}/kfctl.yaml"))}"
+    sak_kustomize_hash = "${data.external.trigger.result["checksum"]}"
+  }
+
   depends_on = [
     aws_iam_role.this,
     aws_rds_cluster.db
   ]
   provisioner "local-exec" {
-    command = "cd ${path.module} && kfctl build -f kfctl.yaml"
+    command = "cd ${path.module} && rm -rf .cache && rm -rf kustomize && kfctl build -f kfctl.yaml"
   }
 }
 
 resource "null_resource" "sak_kustomize" {
+  triggers = {
+    kfctl_build = null_resource.kfctl_build.id
+  }
   depends_on = [
     null_resource.kfctl_build
   ]

prerequisites_install.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+brew install helm kubernetes-cli awscli aws-iam-authenticator terraform jq
+echo
+echo "Installed helm version:"
+helm version
+echo
+echo "Installed kubectl version:"
+kubectl version --client
+echo
+echo "Installed awscli version:"
+aws --version
+echo
+echo "Installed aws-iam-authenticator version:"
+aws-iam-authenticator version
+echo
+echo "Installed terraform version:"
+terraform version | head -n 1
+echo
+echo "Installed jq version:"
+jq --version
+echo
+
+bash "$SCRIPTPATH/kfctl_install.sh"