diff --git a/.github/workflows/knative-e2e-tests.yaml b/.github/workflows/knative-e2e-tests.yaml index fd7033989..7b52cf463 100644 --- a/.github/workflows/knative-e2e-tests.yaml +++ b/.github/workflows/knative-e2e-tests.yaml @@ -74,12 +74,17 @@ jobs: - uses: azure/setup-helm@v1 + - name: RBAC rules for helm + run: | + export ITER8=$(pwd) + kubectl apply -f ${ITER8}/samples/knative/canaryprogressive/helm-rbac.yaml + - name: create Knative app with canary run: | export ITER8=$(pwd) - helm install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --namespace=iter8-system + helm install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --namespace=default kubectl wait ksvc/sample-app --for condition=Ready --timeout=240s - helm upgrade --install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --values=https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/experimental-values.yaml --namespace=iter8-system + helm upgrade --install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --values=https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/experimental-values.yaml --namespace=default - name: Generate requests run: | diff --git a/mkdocs/docs/getting-started/install.md b/mkdocs/docs/getting-started/install.md index 07ceea263..f4ee16d5f 100644 --- a/mkdocs/docs/getting-started/install.md +++ b/mkdocs/docs/getting-started/install.md @@ -15,63 +15,19 @@ title: Installation Install Iter8 in your Kubernetes cluster as follows. ```shell -export TAG=v0.3.0 +export TAG=v0.3.2 curl -s https://raw.githubusercontent.com/iter8-tools/iter8-install/main/install.sh | bash ``` -??? info "Look inside install.sh" - ```shell - #!/bin/bash - - set -e - - # Step 0: Export TAG - export TAG="${TAG:-v0.3.0}" - - # Step 1: Install Iter8 - echo "Installing Iter8" - kubectl apply -f https://raw.githubusercontent.com/iter8-tools/iter8-install/${TAG}/core/build.yaml - kubectl wait crd -l creator=iter8 --for condition=established --timeout=120s - kubectl apply -f https://raw.githubusercontent.com/iter8-tools/iter8-install/${TAG}/metrics/build.yaml - - echo "Verifying Iter8 installation" - kubectl wait --for condition=ready --timeout=300s pods --all -n iter8-system - - set +e - ``` - ## (Optional) Step 2: Prometheus add-on Install Iter8's Prometheus add-on in your cluster as follows. This step assumes you have installed Iter8 following Step 1 above. ```shell -export TAG=v0.3.0 +export TAG=v0.3.2 curl -s https://raw.githubusercontent.com/iter8-tools/iter8-install/main/install-prom-add-on.sh | bash ``` -??? info "Look inside install-prom-add-on.sh" - ```shell - #!/bin/bash - - set -e - - # Step 0: Export TAG - export TAG="${TAG:-v0.3.0}" - - # Step 1: Install Prometheus add-on - # This step assumes you have installed Iter8 using install.sh - echo "Installing Prometheus add-on" - kubectl apply -f https://raw.githubusercontent.com/iter8-tools/iter8-install/${TAG}/prometheus-add-on/prometheus-operator/build.yaml - kubectl wait crd -l creator=iter8 --for condition=established --timeout=120s - kubectl apply -f https://raw.githubusercontent.com/iter8-tools/iter8-install/${TAG}/prometheus-add-on/prometheus/build.yaml - kubectl apply -f https://raw.githubusercontent.com/iter8-tools/iter8-install/${TAG}/prometheus-add-on/service-monitors/build.yaml - - echo "Verifying Prometheus-addon installation" - kubectl wait --for condition=ready --timeout=300s pods --all -n iter8-system - - set +e - ``` - ??? note "Running Iter8 tutorials without Iter8's Prometheus add-on" When you installed Iter8 in the first step above, you also installed several *out-of-the-box* Iter8 metric resources. They are required for running the tutorials documented on this site. diff --git a/mkdocs/docs/getting-started/quick-start/with-knative.md b/mkdocs/docs/getting-started/quick-start/with-knative.md index 840a403da..6bfcb0ce9 100644 --- a/mkdocs/docs/getting-started/quick-start/with-knative.md +++ b/mkdocs/docs/getting-started/quick-start/with-knative.md @@ -56,6 +56,8 @@ export ITER8=$(pwd) ## 3. Install Knative and Iter8 Knative can work with multiple networking layers. So can Iter8's Knative extension. Choose a networking layer for Knative. +This also installs sample metrics for Knative and Iter8's Prometheus add-on. These optional components are used by the sample experiment below. + === "Contour" ```shell @@ -194,8 +196,9 @@ kubectl apply -f $ITER8/samples/knative/quickstart/experiment.yaml strategy: # this experiment will perform a canary test testingPattern: Canary + deploymentPattern: Progressive actions: - start: # run a sequence of tasks at the start of the experiment + start: # run the following sequence of tasks at the start of the experiment - task: knative/init-experiment finish: # run the following sequence of tasks at the end of the experiment - task: common/exec # promote the winning version @@ -206,6 +209,7 @@ kubectl apply -f $ITER8/samples/knative/quickstart/experiment.yaml - "-f" - "https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/quickstart/{{ .promote }}.yaml" criteria: + requestCount: iter8-knative/request-count # mean latency of version should be under 50 milliseconds # 95th percentile latency should be under 100 milliseconds # error rate should be under 1% @@ -221,22 +225,20 @@ kubectl apply -f $ITER8/samples/knative/quickstart/experiment.yaml iterationsPerLoop: 10 versionInfo: # information about app versions used in this experiment - baseline: - name: current - variables: - # variables are used when querying metrics and when interpolating task inputs - - name: revision - value: sample-app-v1 - - name: promote - value: baseline - candidates: - - name: candidate - variables: - # variables are used when querying metrics and when interpolating task inputs - - name: revision - value: sample-app-v2 - - name: promote - value: candidate + baseline: + name: current + variables: + - name: revision + value: sample-app-v1 + - name: promote + value: baseline + candidates: + - name: candidate + variables: + - name: revision + value: sample-app-v2 + - name: promote + value: candidate ``` The process automated by Iter8 during this experiment is depicted below. diff --git a/mkdocs/docs/tutorials/knative/canary-fixedsplit.md b/mkdocs/docs/tutorials/knative/canary-fixedsplit.md index bf8359bce..9103e6dc5 100644 --- a/mkdocs/docs/tutorials/knative/canary-fixedsplit.md +++ b/mkdocs/docs/tutorials/knative/canary-fixedsplit.md @@ -19,7 +19,7 @@ You will create the following resources in this tutorial. - eventually replaces `baseline` with `candidate` using Kustomize ???+ warning "Before you begin, you will need... " - **Kubernetes cluster:** Ensure that you have Kubernetes cluster with Iter8 and Knative installed. You can do this by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/). + **Kubernetes cluster:** Ensure that you have a Kubernetes cluster with Iter8, Knative, Iter8 sample metrics for Knative, and Prometheus installed. You can do this by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/). **Cleanup:** If you ran an Iter8 tutorial earlier, run the associated cleanup step. diff --git a/mkdocs/docs/tutorials/knative/canary-progressive.md b/mkdocs/docs/tutorials/knative/canary-progressive.md index a2db57fcb..4fc8a63dc 100644 --- a/mkdocs/docs/tutorials/knative/canary-progressive.md +++ b/mkdocs/docs/tutorials/knative/canary-progressive.md @@ -19,7 +19,7 @@ You will create the following resources in this tutorial. - eventually replaces `baseline` with `candidate` using Helm ???+ warning "Before you begin, you will need... " - **Kubernetes cluster:** Ensure that you have Kubernetes cluster with Iter8 and Knative installed. You can do this by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/). + **Kubernetes cluster:** Ensure that you have a Kubernetes cluster with Iter8, Knative, Iter8 sample metrics for Knative, and Prometheus installed. You can do this by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/). **Cleanup:** If you ran an Iter8 tutorial earlier, run the associated cleanup step. @@ -27,11 +27,19 @@ You will create the following resources in this tutorial. **[Helm v3](https://helm.sh/) and [`iter8ctl`](../../../getting-started/install/#optional-step-3-iter8ctl):** This tutorial uses Helm v3 and `iter8ctl`. -## 1. Create versions +## 1. Give Permission to Iter8 to Call `helm upgrade` + +Helm uses secrets to record information about releases. This tutorial uses an experiment that invokes Helm. Enable this experiment using the following RBAC. + +```shell +kubectl apply -f ${ITER8}/samples/knative/canaryprogressive/helm-rbac.yaml +``` + +## 2. Create versions ```shell -helm install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --namespace=iter8-system +helm install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --namespace=default kubectl wait ksvc/sample-app --for condition=Ready --timeout=120s -helm upgrade --install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --values=https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/experimental-values.yaml --namespace=iter8-system +helm upgrade --install --repo https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo sample-app sample-app --values=https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/experimental-values.yaml --namespace=default ``` ??? info "Look inside values.yaml" @@ -60,7 +68,7 @@ helm upgrade --install --repo https://raw.githubusercontent.com/iter8-tools/iter percent: 0 ``` -## 2. Generate requests +## 3. Generate requests ```shell kubectl wait --for=condition=Ready ksvc/sample-app URL_VALUE=$(kubectl get ksvc sample-app -o json | jq .status.address.url) @@ -98,7 +106,7 @@ sed "s+URL_VALUE+${URL_VALUE}+g" $ITER8/samples/knative/canaryprogressive/fortio restartPolicy: Never ``` -## 3. Create Iter8 experiment +## 4. Create Iter8 experiment ```shell kubectl apply -f $ITER8/samples/knative/canaryprogressive/experiment.yaml ``` @@ -134,7 +142,7 @@ kubectl apply -f $ITER8/samples/knative/canaryprogressive/experiment.yaml - "--repo" - "https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo" # repo url - "sample-app" # release name - - "--namespace=iter8-system" # release namespace + - "--namespace=default" # release namespace - "sample-app" # chart name - "--values=https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/{{ .promote }}-values.yaml" # values URL dynamically interpolated criteria: @@ -152,7 +160,7 @@ kubectl apply -f $ITER8/samples/knative/canaryprogressive/experiment.yaml intervalSeconds: 10 iterationsPerLoop: 10 versionInfo: - # information about versions used in this experiment + # information about app versions used in this experiment baseline: name: current variables: @@ -166,10 +174,10 @@ kubectl apply -f $ITER8/samples/knative/canaryprogressive/experiment.yaml - name: revision value: sample-app-v2 - name: promote - value: candidate + value: candidate ``` -## 4. Observe experiment +## 5. Observe experiment Observe the experiment in realtime. Paste commands from the tabs below in separate terminals. === "iter8ctl" @@ -203,16 +211,16 @@ Observe the experiment in realtime. Paste commands from the tabs below in separa As the experiment progresses, you should see traffic progressively shift from `sample-app-v1` to `sample-app-v2`. When the experiment completes, all of the traffic will be sent to the winner, `sample-app-v2`. -## 5. Cleanup +## 6. Cleanup ```shell kubectl delete -f $ITER8/samples/knative/canaryprogressive/experiment.yaml kubectl delete -f $ITER8/samples/knative/canaryprogressive/fortio.yaml -helm uninstall sample-app --namespace=iter8-system +helm uninstall sample-app --namespace=default ``` ???+ info "Understanding what happened" 1. You created a Knative service using `helm install` subcommand and upgraded the service to have two revisions, sample-app-v1 (`baseline`) and sample-app-v2 (`candidate`) using `helm upgrade --install` subcommand. - 2. The ksvc is created in the `default` namespace. Helm release information is located in the `iter8-system` namespace as specified by the `--namespace=iter8-system` flag. + 2. The ksvc is created in the `default` namespace. Likewise, the Helm release information is located in the `default` namespace as specified by the `--namespace=default` flag. 3. You generated requests for the Knative service using a Fortio job. At the start of the experiment, 100% of the requests are sent to baseline and 0% to candidate. 4. You created an Iter8 `Canary` experiment with `Progressive` deployment pattern. In each iteration, Iter8 observed the mean latency, 95th percentile tail-latency, and error-rate metrics collected by Prometheus, verified that `candidate` satisfied all the objectives specified in the experiment, identified `candidate` as the `winner`, progressively shifted traffic from `baseline` to `candidate` and eventually promoted the `candidate` using `helm upgrade --install` subcommand. - **Note:** Had `candidate` failed to satisfy `objectives`, then `baseline` would have been promoted. diff --git a/mkdocs/docs/tutorials/knative/conformance.md b/mkdocs/docs/tutorials/knative/conformance.md index 2fbe7bec5..214fa7725 100644 --- a/mkdocs/docs/tutorials/knative/conformance.md +++ b/mkdocs/docs/tutorials/knative/conformance.md @@ -16,7 +16,7 @@ You will create the following resources in this tutorial. 3. An **Iter8 experiment** that verifies that `baseline` satisfies mean latency, 95th percentile tail latency, and error rate `objectives`. ???+ warning "Before you begin, you will need... " - **Kubernetes cluster:** Ensure that you have Kubernetes cluster with Iter8 and Knative installed. You can do this by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/). + **Kubernetes cluster:** Ensure that you have a Kubernetes cluster with Iter8, Knative, Iter8 sample metrics for Knative, and Prometheus installed. You can do this by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/). **Cleanup:** If you ran an Iter8 tutorial earlier, run the associated cleanup step. diff --git a/mkdocs/docs/tutorials/knative/mirroring.md b/mkdocs/docs/tutorials/knative/mirroring.md index 27fb9bb39..546353a8b 100644 --- a/mkdocs/docs/tutorials/knative/mirroring.md +++ b/mkdocs/docs/tutorials/knative/mirroring.md @@ -17,7 +17,7 @@ You will create the following resources in this tutorial. 4. An **Iter8 experiment** that verifies that the dark version satisfies mean latency, 95th percentile tail latency, and error rate `objectives`. ???+ warning "Before you begin, you will need... " - **Kubernetes cluster with Iter8, Knative and Istio:** Ensure that you have Kubernetes cluster with Iter8 and Knative installed, and that Knative uses the Istio networking layer. You can do so by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/), and selecting Istio during Step 3. + **Kubernetes cluster with Iter8, Knative and Istio:** Ensure that you have a Kubernetes cluster with Iter8, Knative with the Istio networking layer, Iter8 sample metrics for Knative, and Prometheus installed. You can do so by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/), and selecting Istio during Step 3. **Cleanup:** If you ran an Iter8 tutorial earlier, run the associated cleanup step. diff --git a/mkdocs/docs/tutorials/knative/traffic-segmentation.md b/mkdocs/docs/tutorials/knative/traffic-segmentation.md index cead5dbc2..82a85469c 100644 --- a/mkdocs/docs/tutorials/knative/traffic-segmentation.md +++ b/mkdocs/docs/tutorials/knative/traffic-segmentation.md @@ -17,7 +17,7 @@ You will create the following resources in this tutorial. 4. An **Iter8 experiment** which verifies that the `candidate` satisfies mean latency, 95th percentile tail latency, and error rate `objectives`, and progressively increases the proportion of traffic with `country: wakanda` header that is routed to the `candidate`. ???+ warning "Before you begin, you will need... " - **Kubernetes cluster with Iter8, Knative and Istio:** Ensure that you have Kubernetes cluster with Iter8 and Knative installed, and that Knative uses the Istio networking layer. You can do this by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/), and selecting Istio during Step 3. + **Kubernetes cluster with Iter8, Knative and Istio:** Ensure that you have a Kubernetes cluster with Iter8, Knative with the Istio networking layer, Iter8 sample metrics for Knative, and Prometheus installed. You can do this by following Steps 1, 2, and 3 of the [quick start tutorial for Knative](../../../getting-started/quick-start/with-knative/), and selecting Istio during Step 3. **Cleanup:** If you ran an Iter8 tutorial earlier, run the associated cleanup step. diff --git a/samples/knative/canaryprogressive/check.sh b/samples/knative/canaryprogressive/check.sh index 32766469f..cfcf259fd 100755 --- a/samples/knative/canaryprogressive/check.sh +++ b/samples/knative/canaryprogressive/check.sh @@ -2,23 +2,39 @@ set -e +EXPERIMENT=canary-progressive + +# dump logs from iter8 pods +dump() { + # dump handler logs + for pod in $(kubectl -n iter8-system get po --selector=iter8/experimentName=${EXPERIMENT} -o jsonpath='{.items[*].metadata.name}'); do + kubectl -n iter8-system logs $pod + done + + # dump controller logs + kubectl -n iter8-system logs $(kubectl -n iter8-system get po --selector=control-plane=controller-manager -o jsonpath='{.items[0].metadata.name}') + + # dump analytics logs + kubectl -n iter8-system logs $(kubectl -n iter8-system get po --selector=app=iter8-analytics -o jsonpath='{.items[0].metadata.name}') +} + # Ensure ITER8 environment variable is set if [[ -z ${ITER8} ]]; then echo "ITER8 environment variable needs to be set to the root folder of Iter8" - exit 1 + dump; exit 1 else echo "ITER8 is set to " $ITER8 fi # Check if experiment has completed completed="Completed" -stage=$(kubectl get experiment canary-progressive -o json | jq -r .status.stage) +stage=$(kubectl get experiment ${EXPERIMENT} -o json | jq -r .status.stage) if [[ $stage = $completed ]]; then echo "Experiment has Completed" else echo "Experiment must be $completed;" echo "Experiment is $stage;" - exit 1 + dump; exit 1 fi # Check if no packets have been lost by Fortio @@ -33,17 +49,17 @@ else echo "Packets were lost" echo "total requests:" $REQUESTSTOTAL echo "200 requests:" $REQUESTS200 - exit 1 + dump; exit 1 fi # Check if versionRecommendedForPromotion is candidate candidate="candidate" -vrfp=$(kubectl get experiment canary-progressive -o json | jq -r .status.versionRecommendedForPromotion) +vrfp=$(kubectl get experiment ${EXPERIMENT} -o json | jq -r .status.versionRecommendedForPromotion) if [[ $vrfp = $candidate ]]; then echo "versionRecommendedForPromotion is $vrfp" else echo "versionRecommendedForPromotion must be candidate; is" $vrfp - exit 1 + dump; exit 1 fi # Check if latest revision is true @@ -53,6 +69,7 @@ if [[ $lrStatus = $latestRevision ]]; then echo "latestRevision is true" else echo "latestRevision must be true; is" $lrStatus + dump; exit 1 exit 1 fi @@ -63,7 +80,7 @@ if [[ $actualPercent -eq $percent ]]; then echo "percent is 100" else echo "percent must be 100; is" $percent - exit 1 + dump; exit 1 fi set +e diff --git a/samples/knative/canaryprogressive/experiment.yaml b/samples/knative/canaryprogressive/experiment.yaml index 53ec550a1..00110b8bb 100644 --- a/samples/knative/canaryprogressive/experiment.yaml +++ b/samples/knative/canaryprogressive/experiment.yaml @@ -27,7 +27,7 @@ spec: - "--repo" - "https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/helm-repo" # repo url - "sample-app" # release name - - "--namespace=iter8-system" # release namespace + - "--namespace=default" # release namespace - "sample-app" # chart name - "--values=https://raw.githubusercontent.com/iter8-tools/iter8/master/samples/knative/canaryprogressive/{{ .promote }}-values.yaml" # values URL dynamically interpolated criteria: diff --git a/samples/knative/canaryprogressive/helm-rbac.yaml b/samples/knative/canaryprogressive/helm-rbac.yaml new file mode 100644 index 000000000..b3e3513be --- /dev/null +++ b/samples/knative/canaryprogressive/helm-rbac.yaml @@ -0,0 +1,53 @@ +# This role enables writing secrets; +# Iter8 handler uses this role during Helm release tasks +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: iter8-secret-writer +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: ["create", "update", "patch", "delete"] +--- +# Helm v3 uses secrets to store Helm release information. +# +# By default, when Helm is used within Iter8 experiments, secrets +# will be located in the default namespace. +# +# This role binding enables Iter8 handler to read +# secrets in the default namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: secret-reader-handler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: iter8-secret-reader +subjects: +- kind: ServiceAccount + name: iter8-handlers + namespace: iter8-system +--- +# Helm v3 uses secrets to store Helm release information. +# +# By default, when Helm is used within Iter8 experiments, secrets +# will be located in the default namespace. +# +# This role binding enables Iter8 handler to write +# secrets in the default namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: secret-writer-handler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: iter8-secret-writer +subjects: +- kind: ServiceAccount + name: iter8-handlers + namespace: iter8-system +--- diff --git a/samples/knative/quickstart/platformsetup.sh b/samples/knative/quickstart/platformsetup.sh index 024ee3263..6db0a8f46 100755 --- a/samples/knative/quickstart/platformsetup.sh +++ b/samples/knative/quickstart/platformsetup.sh @@ -30,7 +30,7 @@ if [[ ! " ${NETWORK_LAYERS[@]} " =~ " ${1} " ]]; then fi # Step 1: Export correct tags for install artifacts -export TAG="${TAG:-v0.3.0}" +export TAG="${TAG:-v0.3.2}" export KNATIVE_TAG="${KNATIVE_TAG:-v0.21.0}" echo "TAG = $TAG" echo "KNATIVE_TAG = $KNATIVE_TAG" @@ -114,7 +114,7 @@ fi ### Note: the preceding steps perform domain install; following steps perform Iter8 install # Step 5: Install Iter8 -echo "Installing Iter8" +echo "Installing Iter8 with Knative support" curl -s https://raw.githubusercontent.com/iter8-tools/iter8-install/main/install.sh | bash # Step 6: Install Iter8's Prometheus add-on