Finalizes post-setup script and updates docs

.gitignore

@@ -61,3 +61,5 @@ test/tracetesting/tracetesting-vars.yaml
 
 # RCA demo 2024
 /scripts/rca-demo/.env
+/scripts/rca-demo/.env.*
+!/scripts/rca-demo/.env.example

scripts/rca-demo/.env.example

@@ -9,8 +9,6 @@
 #
 ############################################################################
 
-
-
 # NOTE: http://host.minikube.internal and http://host.docker.internal are BOTH
 # valid domains for reaching your machine's localhost from inside the minikube container.
 #
@@ -24,7 +22,7 @@
 # APM server instance or another OTel collector endpoint
 #
 # e.g. "http://host.minikube.internal:8200"
-export OTEL_EXPORTER_OTLP_ENDPOINT=""
+export OTEL_EXPORTER_OTLP_ENDPOINT="http://host.minikube.internal:8200"
 
 # OTEL_EXPORTER_OTLP_HEADERS
 #
@@ -35,7 +33,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT=""
 # However, if your OTLP endpoint requires no authorization, these headers are still
 # expected by the script, even though they will be ignored by the OTLP collector.
 # For that reason, just use "Authorization=Bearer none" to bypass this requirement.
-export OTEL_EXPORTER_OTLP_HEADERS=""
+export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer none"
 
 # OTEL_DEMO_ES_ENDPOINT
 #
@@ -44,64 +42,33 @@ export OTEL_EXPORTER_OTLP_HEADERS=""
 # Kibana is configured to connect to, so that the demo works across all products.
 #
 # e.g. http://host.minikube.internal:9200
-export OTEL_DEMO_ES_ENDPOINT=""
+export OTEL_DEMO_ES_ENDPOINT="http://host.minikube.internal:9200"
 
 # OTEL_DEMO_ES_API_KEY
 #
 # Your Elasticsearch instance is likely set up with security enabled, which means
 # you will need an ES API key to interact with it. For help on creating an
 # API key, visit https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html
+#
+# NOTE: This should be the Base64 encoded version that is returned as "encoded" 
+# from the create API endpoint, or as the only value in the UI.
 export OTEL_DEMO_ES_API_KEY=""
 
-# SYNTHETICS_API_KEY
+# TLS_INSECURE
 #
-# DO NOT SET THIS if you don't want to set up synthetics. Leave it as an empty string
-# to opt out of all synthetics set up.
-
-# Otherwise, get a synthetics API key from the "Project API Key" 
-# page in your Synthetics Kibana app
-export SYNTHETICS_API_KEY="YOURSYNTHETICSAPIKEYI"
+# For most local dev scenarios, you need to specify tls.insecure: true in the
+# collector config (in both kubernetes/elastic-helm/deployment.yaml and in 
+# kubernetes/elastic-helm/daemonset.yaml) — this will do that for you.
+export TLS_INSECURE="true"
 
 # KIBANA_URL
 #
-# Only required if opting into synthetics set up.
-#
-# URL where your connected Kibana is running. If you are running Kibana locally,
-# this will probabably be http://host.minikube.internal:5601/basepath or similar.
-#
-# Fuller explanation TBA
-export KIBANA_URL=""
-
-# KIBANA_API_KEY
-#
-# Only required if opting into synthetics set up.
-#
-# Explanation TBA
-export KIBANA_API_KEY=""
-
-# OTEL_DEMO_FLEET_URL
-#
-# Only required if opting into synthetics set up.
-#
-# Create a fleet policy (for a synthetics private location) and go through the "add agent" flow for it. 
-# Within the policy go through the add agent flow and get these values from there. 
-# 
-# Note: for local dev, you will need to set up a Fleet SERVER before you can extract the agent setup values.
-#
-# Fuller explanation TBA
-export OTEL_DEMO_FLEET_URL=""
-
-# OTEL_DEMO_FLEET_ENROLLMENT_TOKEN
-#
-# Only required if opting into synthetics set up.
-#
-# Explanation TBA
-export OTEL_DEMO_FLEET_ENROLLMENT_TOKEN=""
-
-
+# URL where your connected Kibana is running (with basepath)
+export KIBANA_URL="http://localhost:5601/local"
 
-# SYNTHETICS_PRIVATE_LOCATION
+# KIBANA_USER and KIBANA_PASSWORD
 #
-# Make sure to create a synthetics private location in the synthetics UI linked to the previously 
-# created fleet policy. It should generally be named otel-demo, but you can change it here.
-export SYNTHETICS_PRIVATE_LOCATION="otel-demo"
+# Provide a user and password combo that can be used to interact with
+# Kibana REST APIs, this user should have "admin" access.
+export KIBANA_USER="elastic"
+export KIBANA_PASSWORD="changeme"

scripts/rca-demo/README.md

@@ -1,34 +1,45 @@
 # RCA KubeCon Demo Scripts
 
-Docs for running scripts in this folder. 
+Docs for running scripts in this folder. It is highly recommended to run these scripts while you are in the root of the repo, e.g. `./scripts/rca-demo/{script_name}`
 
+### Environment variables
 
-## setup
+Please be sure to check the latest docs in the `.env.example` file in this folder to explain the required env vars needed for these scripts to run.
+
+Most of these scripts require env vars that can be set most easily by creating a local ".env" file in this directory (it will be ignored by git and auto-sourced by the script). You can create more .env.* files and they will also be ignored. To change which file gets auto-sourced, use:
+
+```
+ENV_FILE_PATH=./scripts/rca-demo/.env.{suffix} ./scripts/rca-demo/{script_name}
+```
 
-The fast way to set things up is with the `setup` script. Before running it, be sure to copy 
-[the .env.example file](./.env.example) and add all of the required values. Each script file SHOULD
-source this .env file for you, but if it doesn't, run `$ source scripts/rca-demo/.env` to make sure.
+## setup
 
-Run the setup script from the repo root:
+This will primarily set up **the k8s otel demo cluster**.
 
 ```
 ./scripts/rca-demo/setup
 ```
 
-The output will guide you if you are missing anything required to make this work. You can check the
-status of the pods by running `kubectl get pods`. 
+The output will guide you if you are missing anything required to make this work. You can check the status of the pods by running `kubectl get pods`. 
 
-Assuming everything is now running correctly, you should be able to visit the Astronomy Shop in
-your browser at http://otel-demo.internal, and see signals in your configured ES/Kibana.
+Assuming everything is now running correctly, you should be able to visit the Astronomy Shop in your browser at http://otel-demo.internal, and see signals in your configured ES/Kibana.
 
-## trigger-demo scenario
+## post-setup
 
-The `trigger-demo-scenario` script will cause the cart service to fail to start properly. You can use this to test how the solution responds to the problem.
+This script sets up assets in Kibana and Elasticsearch related to this demo, without needing to do the full kubernetes + helm install process. It will install alerting rules, data views, mapping updates, and advanced configuration settings required for the demo to work. Many of the definitions used for creating these assets can be found in the `scripts/rca-demo/data` directory.
 
-For the full scenario, set up a custom threshold rule like this:
+```
+./scripts/rca-demo/post-setup
+```
+
+Remember: you can create more .env.other files with different sets of values to run this script against a different ES/Kibana set up (e.g. keep .env for your local set up and .env.cloud for a cloud setup, etc.). To auto-source a different env file, run this like this:
+
+```
+ENV_FILE_PATH=./scripts/rca-demo/.env.cloud ./scripts/rca-demo/post-setup
+```
 
-![Custom threshold rule](threshold_rule.png "Custom threshold rule")
+## trigger-demo scenario
 
-This rule will trigger when the demo scenario is activated and will be associated with the nginx ingress controller service. It can be used as a starting point to showcase the different exploration capabilities of the stack.
+The `trigger-demo-scenario` script will cause the cart service to fail to start properly. You can use this to test how the solution responds to the problem.
 
-With `trigger-demo-scenario restore`, the system can be put back into a working state again.
+Use `trigger-demo-scenario restore` to put the cart service back into a working state again.

scripts/rca-demo/create-release-annotation

@@ -33,8 +33,12 @@ if [ -z "${KIBANA_URL}" ]; then
   die "You must set KIBANA_URL to a valid Kibana URL"
 fi
 
-if [ -z "${KIBANA_API_KEY}" ]; then
-  die "You must set KIBANA_API_KEY to a valid KIBANA API KEY"
+if [ -z "${KIBANA_USER}" ]; then
+  die "You must set KIBANA_USER to a valid admin Kibana user"
+fi
+
+if [ -z "${KIBANA_PASSWORD}" ]; then
+  die "You must set KIBANA_PASSWORD for the specified user"
 fi
 
 if [[ "$1" != -* ]]; then
@@ -79,11 +83,10 @@ EOF
 HEADERS=(
     -H "kbn-xsrf: true" \
     -H "elastic-api-version: 2023-10-31" \
-    -H "Content-Type: application/json" \
-    -H "Authorization: $KIBANA_API_KEY"
+    -H "Content-Type: application/json"
 )
 
-response=$(curl -s -X POST "${HEADERS[@]}" -d "$REQUEST_BODY" "$ANNOTATION_API_ENDPOINT")
+response=$(curl -s -X POST -u "$KIBANA_USER:$KIBANA_PASSWORD" "${HEADERS[@]}" -d "$REQUEST_BODY" "$ANNOTATION_API_ENDPOINT")
 
 statusCode=$(echo "$response" | jq -r '.statusCode')
 ok=$(echo "$response" | jq -r '.ok')

scripts/rca-demo/data/data_view-01.json

@@ -0,0 +1,8 @@
+{
+  "data_view": { 
+    "name": "logs_otel_test_cli (Automated by Demo CLI)", 
+    "title": "logs-*otel*", 
+    "id": "logs_otel_test_cli",
+    "timeFieldName": "@timestamp" 
+  }
+}

scripts/rca-demo/data/rule-01.json

@@ -31,7 +31,7 @@
         "query": "k8s.namespace.name: \"ingress-nginx\" AND url.path: /api/*",
         "language": "kuery"
       },
-      "index": "otel_logs_data"
+      "index": "apm_static_data_view_id_default"
     },
     "groupBy": [
       "service.name",

scripts/rca-demo/post-setup

@@ -1,21 +1,30 @@
 #!/bin/bash
 
 SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}")
+LOAD_ENV_FILE_PATH=$SCRIPTS_DIR/.env
 
 set -a
 source $SCRIPTS_DIR/shared.sh
-source $SCRIPTS_DIR/.env
 set +a
 
+if [ -n "${ENV_FILE_PATH}" ]; then
+  LOAD_ENV_FILE_PATH=$ENV_FILE_PATH
+fi
+
+title "Sourcing environment variables from $LOAD_ENV_FILE_PATH"
+source $LOAD_ENV_FILE_PATH
+
 create_data_view () {
 	ID=$1
 	INDICES=$2
 
-	EXISTS=$(curl -s $KIBANA_URL/api/data_views/data_view/$ID -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H 'elastic-api-version: 2023-10-31')
+	EXISTS=$(curl -s $KIBANA_URL/api/data_views/data_view/$ID -u "$KIBANA_USER:$KIBANA_PASSWORD" -H 'elastic-api-version: 2023-10-31')
+
+	title "Attempting to create data view $ID"
 
 	if [[ $EXISTS == '{"statusCode":404,"error":"Not Found"'* ]]; then
 		title "Creating data view $1 for indices $2"
-		curl -X POST $KIBANA_URL/api/data_views/data_view -H 'kbn-xsrf: bananas' -H 'Content-Type: application/json' -H 'elastic-api-version: 2023-10-31' -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -d "{\"data_view\": { \"name\": \"$1 (Automated by Demo CLI)\", \"title\": \"$2\", \"id\": \"$1\", \"timeFieldName\": \"@timestamp\" }}"
+		curl -X POST $KIBANA_URL/api/data_views/data_view -u "$KIBANA_USER:$KIBANA_PASSWORD" -H 'kbn-xsrf: bananas' -H 'Content-Type: application/json' -H 'elastic-api-version: 2023-10-31' -d "{\"data_view\": { \"name\": \"$1 (Automated by Demo CLI)\", \"title\": \"$2\", \"id\": \"$1\", \"timeFieldName\": \"@timestamp\" }}"
 	else
 		title "Data view $1 already exists"
 	fi
@@ -24,8 +33,10 @@ create_data_view () {
 create_rule () {
 	ID=$1
 	DATA_FILE=$2
+
+	title "Attempting to create rule from $DATA_FILE"
 
-	curl -X POST $KIBANA_URL/api/alerting/rule/$ID -H 'kbn-xsrf: bananas' -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H 'Content-Type: application/json; elastic-api-version: 2023-10-31' -d @$SCRIPTS_DIR/data/$DATA_FILE
+	curl -X POST $KIBANA_URL/api/alerting/rule/$ID -u "$KIBANA_USER:$KIBANA_PASSWORD" -H 'kbn-xsrf: bananas' -H 'Content-Type: application/json; elastic-api-version: 2023-10-31' -d @$SCRIPTS_DIR/data/$DATA_FILE
 }
 
 title "Setting up Kibana assets for the demo..."
@@ -34,12 +45,27 @@ if [ -z "${KIBANA_URL}" ]; then
   die "You must set KIBANA_URL so that we can post API requests there (please include base path if one exists)"
 fi
 
+if [ -z "${OTEL_DEMO_ES_ENDPOINT}" ]; then
+  die "You must set OTEL_DEMO_ES_ENDPOINT"
+fi
+
 if [ -z "${OTEL_DEMO_ES_API_KEY}" ]; then
-  die "You must set OTEL_DEMO_ES_API_KEY (must be valid API key for ES and Kibana APIs)"
+  die "You must set OTEL_DEMO_ES_API_KEY (must be valid API key for ES APIs)"
 fi
 
+if [ -z "${KIBANA_USER}" ]; then
+  die "You must set KIBANA_USER to an admin user"
+fi
+
+if [ -z "${KIBANA_PASSWORD}" ]; then
+  die "You must set KIBANA_PASSWORD for the specified user"
+fi
+
+# Replace kubernetes internal host pattern with localhost
+OTEL_DEMO_ES_ENDPOINT=$(echo $OTEL_DEMO_ES_ENDPOINT | sed "s/host.minikube.internal/localhost/" | sed "s/host.docker.internal/localhost/")
+
 # Create data views
-create_data_view otel_logs_data logs-*otel* 
+# create_data_view otel_logs_data logs-*otel* 
 
 # Create rules
 create_rule 9055220c-8fb1-4f9f-be7c-0a33eb2bafc5 rule-01.json
@@ -50,10 +76,10 @@ echo -e "\n"
 # NOTE: this will fail if the data stream is not created
 # TODO: Update the component template also, and just swallow errors on this call
 title "Attempting to add message alias for otel logs"
-curl -X PUT http://localhost:9200/logs-*otel*/_mapping -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H "Content-Type: application/json" -d @$SCRIPTS_DIR/data/mapping-message-alias.json
+curl -X PUT "$OTEL_DEMO_ES_ENDPOINT/logs-*otel*/_mapping" -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H "Content-Type: application/json" -d @$SCRIPTS_DIR/data/mapping-message-alias.json
 
 echo -e "\n"
 
 # Enable EEM
 title "Attempting to enable EEM"
-curl -X PUT -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H "kbn-xsrf: bananas" $KIBANA_URL/internal/entities/managed/enablement
+curl -X PUT -u "$KIBANA_USER:$KIBANA_PASSWORD" -H "kbn-xsrf: bananas" -H "x-elastic-internal-origin: Kibana" $KIBANA_URL/internal/entities/managed/enablement

scripts/rca-demo/setup

@@ -10,12 +10,16 @@ set +a
 title "Checking environment for $APP_NAME..."
 
 # Load local user .env file
-ENV_FILE="./scripts/rca-demo/.env"
-if [ -f $ENV_FILE ]; then
-	echo "Sourcing env vars from $ENV_FILE ..."
-  source $ENV_FILE
+LOAD_ENV_FILE_PATH="./scripts/rca-demo/.env"
+
+if [ -n "${ENV_FILE_PATH}" ]; then
+  LOAD_ENV_FILE_PATH=$ENV_FILE_PATH
 fi
 
+if [ -f $LOAD_ENV_FILE_PATH ]; then
+	title "Sourcing environment variables from $LOAD_ENV_FILE_PATH"
+	source $LOAD_ENV_FILE_PATH
+fi
 
 # Check tools
 tools=("helm" "kubectl" "minikube" "node" "npx")
@@ -45,25 +49,6 @@ if [ -z "${OTEL_DEMO_ES_API_KEY}" ]; then
   die "You must set OTEL_DEMO_ES_API_KEY"
 fi
 
-if [ -z "${SYNTHETICS_API_KEY}" ]; then
-	echo "No SYNTHETICS_API_KEY set, will skip synthetics setup"
-else
-  echo "By providing a SYNTHETICS_API_KEY value, you are opting IN to synthetics setup"
-	echo "Please remove this env var or set it to \"\" to opt out."
-
-	if [ -z "${KIBANA_URL}" ]; then
-		die "[Synthetics Requirement] You must set KIBANA_URL"
-	fi
-
-	if [ -z "${OTEL_DEMO_FLEET_URL}" ]; then
-		die "[Synthetics Requirement] You must set OTEL_DEMO_FLEET_URL"
-	fi
-
-	if [ -z "${OTEL_DEMO_FLEET_ENROLLMENT_TOKEN}" ]; then
-		die "[Synthetics Requirement] You must set OTEL_DEMO_FLEET_ENROLLMENT_TOKEN"
-	fi
-fi
-
 : ${TLS_INSECURE:="false"}
 
 if minikube status | grep -q "Running"; then
@@ -148,19 +133,6 @@ pushd_helm_dir
 helminst nginx-ingress ./ingress
 popd
 
-# If synthetics is configured
-if [ -n "${SYNTHETICS_API_KEY}" ]; then
-	title "Installing synthetics agent helm chart"
-	pushd_helm_dir
-	helminst otel-synthetics-agent ./otel-synthetics
-	popd
-
-	title "Pushing Synthetics project..."
-	cd synthetic-monitors
-	npm install
-	yes | npx @elastic/synthetics push . --params '{"url": "http://ingress-nginx-controller.ingress-nginx.svc.cluster.local"}'
-fi
-
 title "Setup complete!"
 cat <<-EOF
 You may access the demo at http://otel-demo.internal