From fed6270132aed427873dad88125b4c370985a736 Mon Sep 17 00:00:00 2001 From: Jason Rhodes Date: Fri, 25 Oct 2024 14:15:49 -0400 Subject: [PATCH] Finalizes post-setup script and updates docs --- .gitignore | 2 + scripts/rca-demo/.env.example | 69 ++++++---------------- scripts/rca-demo/README.md | 43 +++++++++----- scripts/rca-demo/create-release-annotation | 13 ++-- scripts/rca-demo/data/data_view-01.json | 8 +++ scripts/rca-demo/data/rule-01.json | 2 +- scripts/rca-demo/post-setup | 42 ++++++++++--- scripts/rca-demo/setup | 44 +++----------- 8 files changed, 106 insertions(+), 117 deletions(-) create mode 100644 scripts/rca-demo/data/data_view-01.json diff --git a/.gitignore b/.gitignore index 3e1656f591..1e732953dc 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,5 @@ test/tracetesting/tracetesting-vars.yaml # RCA demo 2024 /scripts/rca-demo/.env +/scripts/rca-demo/.env.* +!/scripts/rca-demo/.env.example diff --git a/scripts/rca-demo/.env.example b/scripts/rca-demo/.env.example index ba8581f440..2d68f83ed2 100644 --- a/scripts/rca-demo/.env.example +++ b/scripts/rca-demo/.env.example @@ -9,8 +9,6 @@ # ############################################################################ - - # NOTE: http://host.minikube.internal and http://host.docker.internal are BOTH # valid domains for reaching your machine's localhost from inside the minikube container. # @@ -24,7 +22,7 @@ # APM server instance or another OTel collector endpoint # # e.g. "http://host.minikube.internal:8200" -export OTEL_EXPORTER_OTLP_ENDPOINT="" +export OTEL_EXPORTER_OTLP_ENDPOINT="http://host.minikube.internal:8200" # OTEL_EXPORTER_OTLP_HEADERS # @@ -35,7 +33,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT="" # However, if your OTLP endpoint requires no authorization, these headers are still # expected by the script, even though they will be ignored by the OTLP collector. # For that reason, just use "Authorization=Bearer none" to bypass this requirement. -export OTEL_EXPORTER_OTLP_HEADERS="" +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer none" # OTEL_DEMO_ES_ENDPOINT # @@ -44,64 +42,33 @@ export OTEL_EXPORTER_OTLP_HEADERS="" # Kibana is configured to connect to, so that the demo works across all products. # # e.g. http://host.minikube.internal:9200 -export OTEL_DEMO_ES_ENDPOINT="" +export OTEL_DEMO_ES_ENDPOINT="http://host.minikube.internal:9200" # OTEL_DEMO_ES_API_KEY # # Your Elasticsearch instance is likely set up with security enabled, which means # you will need an ES API key to interact with it. For help on creating an # API key, visit https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html +# +# NOTE: This should be the Base64 encoded version that is returned as "encoded" +# from the create API endpoint, or as the only value in the UI. export OTEL_DEMO_ES_API_KEY="" -# SYNTHETICS_API_KEY +# TLS_INSECURE # -# DO NOT SET THIS if you don't want to set up synthetics. Leave it as an empty string -# to opt out of all synthetics set up. - -# Otherwise, get a synthetics API key from the "Project API Key" -# page in your Synthetics Kibana app -export SYNTHETICS_API_KEY="YOURSYNTHETICSAPIKEYI" +# For most local dev scenarios, you need to specify tls.insecure: true in the +# collector config (in both kubernetes/elastic-helm/deployment.yaml and in +# kubernetes/elastic-helm/daemonset.yaml) — this will do that for you. +export TLS_INSECURE="true" # KIBANA_URL # -# Only required if opting into synthetics set up. -# -# URL where your connected Kibana is running. If you are running Kibana locally, -# this will probabably be http://host.minikube.internal:5601/basepath or similar. -# -# Fuller explanation TBA -export KIBANA_URL="" - -# KIBANA_API_KEY -# -# Only required if opting into synthetics set up. -# -# Explanation TBA -export KIBANA_API_KEY="" - -# OTEL_DEMO_FLEET_URL -# -# Only required if opting into synthetics set up. -# -# Create a fleet policy (for a synthetics private location) and go through the "add agent" flow for it. -# Within the policy go through the add agent flow and get these values from there. -# -# Note: for local dev, you will need to set up a Fleet SERVER before you can extract the agent setup values. -# -# Fuller explanation TBA -export OTEL_DEMO_FLEET_URL="" - -# OTEL_DEMO_FLEET_ENROLLMENT_TOKEN -# -# Only required if opting into synthetics set up. -# -# Explanation TBA -export OTEL_DEMO_FLEET_ENROLLMENT_TOKEN="" - - +# URL where your connected Kibana is running (with basepath) +export KIBANA_URL="http://localhost:5601/local" -# SYNTHETICS_PRIVATE_LOCATION +# KIBANA_USER and KIBANA_PASSWORD # -# Make sure to create a synthetics private location in the synthetics UI linked to the previously -# created fleet policy. It should generally be named otel-demo, but you can change it here. -export SYNTHETICS_PRIVATE_LOCATION="otel-demo" \ No newline at end of file +# Provide a user and password combo that can be used to interact with +# Kibana REST APIs, this user should have "admin" access. +export KIBANA_USER="elastic" +export KIBANA_PASSWORD="changeme" diff --git a/scripts/rca-demo/README.md b/scripts/rca-demo/README.md index 8cc3dc0335..764def03f9 100644 --- a/scripts/rca-demo/README.md +++ b/scripts/rca-demo/README.md @@ -1,34 +1,45 @@ # RCA KubeCon Demo Scripts -Docs for running scripts in this folder. +Docs for running scripts in this folder. It is highly recommended to run these scripts while you are in the root of the repo, e.g. `./scripts/rca-demo/{script_name}` +### Environment variables -## setup +Please be sure to check the latest docs in the `.env.example` file in this folder to explain the required env vars needed for these scripts to run. + +Most of these scripts require env vars that can be set most easily by creating a local ".env" file in this directory (it will be ignored by git and auto-sourced by the script). You can create more .env.* files and they will also be ignored. To change which file gets auto-sourced, use: + +``` +ENV_FILE_PATH=./scripts/rca-demo/.env.{suffix} ./scripts/rca-demo/{script_name} +``` -The fast way to set things up is with the `setup` script. Before running it, be sure to copy -[the .env.example file](./.env.example) and add all of the required values. Each script file SHOULD -source this .env file for you, but if it doesn't, run `$ source scripts/rca-demo/.env` to make sure. +## setup -Run the setup script from the repo root: +This will primarily set up **the k8s otel demo cluster**. ``` ./scripts/rca-demo/setup ``` -The output will guide you if you are missing anything required to make this work. You can check the -status of the pods by running `kubectl get pods`. +The output will guide you if you are missing anything required to make this work. You can check the status of the pods by running `kubectl get pods`. -Assuming everything is now running correctly, you should be able to visit the Astronomy Shop in -your browser at http://otel-demo.internal, and see signals in your configured ES/Kibana. +Assuming everything is now running correctly, you should be able to visit the Astronomy Shop in your browser at http://otel-demo.internal, and see signals in your configured ES/Kibana. -## trigger-demo scenario +## post-setup -The `trigger-demo-scenario` script will cause the cart service to fail to start properly. You can use this to test how the solution responds to the problem. +This script sets up assets in Kibana and Elasticsearch related to this demo, without needing to do the full kubernetes + helm install process. It will install alerting rules, data views, mapping updates, and advanced configuration settings required for the demo to work. Many of the definitions used for creating these assets can be found in the `scripts/rca-demo/data` directory. -For the full scenario, set up a custom threshold rule like this: +``` +./scripts/rca-demo/post-setup +``` + +Remember: you can create more .env.other files with different sets of values to run this script against a different ES/Kibana set up (e.g. keep .env for your local set up and .env.cloud for a cloud setup, etc.). To auto-source a different env file, run this like this: + +``` +ENV_FILE_PATH=./scripts/rca-demo/.env.cloud ./scripts/rca-demo/post-setup +``` -![Custom threshold rule](threshold_rule.png "Custom threshold rule") +## trigger-demo scenario -This rule will trigger when the demo scenario is activated and will be associated with the nginx ingress controller service. It can be used as a starting point to showcase the different exploration capabilities of the stack. +The `trigger-demo-scenario` script will cause the cart service to fail to start properly. You can use this to test how the solution responds to the problem. -With `trigger-demo-scenario restore`, the system can be put back into a working state again. +Use `trigger-demo-scenario restore` to put the cart service back into a working state again. diff --git a/scripts/rca-demo/create-release-annotation b/scripts/rca-demo/create-release-annotation index b489a09eb7..52e0637082 100755 --- a/scripts/rca-demo/create-release-annotation +++ b/scripts/rca-demo/create-release-annotation @@ -33,8 +33,12 @@ if [ -z "${KIBANA_URL}" ]; then die "You must set KIBANA_URL to a valid Kibana URL" fi -if [ -z "${KIBANA_API_KEY}" ]; then - die "You must set KIBANA_API_KEY to a valid KIBANA API KEY" +if [ -z "${KIBANA_USER}" ]; then + die "You must set KIBANA_USER to a valid admin Kibana user" +fi + +if [ -z "${KIBANA_PASSWORD}" ]; then + die "You must set KIBANA_PASSWORD for the specified user" fi if [[ "$1" != -* ]]; then @@ -79,11 +83,10 @@ EOF HEADERS=( -H "kbn-xsrf: true" \ -H "elastic-api-version: 2023-10-31" \ - -H "Content-Type: application/json" \ - -H "Authorization: $KIBANA_API_KEY" + -H "Content-Type: application/json" ) -response=$(curl -s -X POST "${HEADERS[@]}" -d "$REQUEST_BODY" "$ANNOTATION_API_ENDPOINT") +response=$(curl -s -X POST -u "$KIBANA_USER:$KIBANA_PASSWORD" "${HEADERS[@]}" -d "$REQUEST_BODY" "$ANNOTATION_API_ENDPOINT") statusCode=$(echo "$response" | jq -r '.statusCode') ok=$(echo "$response" | jq -r '.ok') diff --git a/scripts/rca-demo/data/data_view-01.json b/scripts/rca-demo/data/data_view-01.json new file mode 100644 index 0000000000..c2fab62609 --- /dev/null +++ b/scripts/rca-demo/data/data_view-01.json @@ -0,0 +1,8 @@ +{ + "data_view": { + "name": "logs_otel_test_cli (Automated by Demo CLI)", + "title": "logs-*otel*", + "id": "logs_otel_test_cli", + "timeFieldName": "@timestamp" + } +} \ No newline at end of file diff --git a/scripts/rca-demo/data/rule-01.json b/scripts/rca-demo/data/rule-01.json index c474bda58c..0c5182ee48 100644 --- a/scripts/rca-demo/data/rule-01.json +++ b/scripts/rca-demo/data/rule-01.json @@ -31,7 +31,7 @@ "query": "k8s.namespace.name: \"ingress-nginx\" AND url.path: /api/*", "language": "kuery" }, - "index": "otel_logs_data" + "index": "apm_static_data_view_id_default" }, "groupBy": [ "service.name", diff --git a/scripts/rca-demo/post-setup b/scripts/rca-demo/post-setup index 05080efa81..e110307d9d 100755 --- a/scripts/rca-demo/post-setup +++ b/scripts/rca-demo/post-setup @@ -1,21 +1,30 @@ #!/bin/bash SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") +LOAD_ENV_FILE_PATH=$SCRIPTS_DIR/.env set -a source $SCRIPTS_DIR/shared.sh -source $SCRIPTS_DIR/.env set +a +if [ -n "${ENV_FILE_PATH}" ]; then + LOAD_ENV_FILE_PATH=$ENV_FILE_PATH +fi + +title "Sourcing environment variables from $LOAD_ENV_FILE_PATH" +source $LOAD_ENV_FILE_PATH + create_data_view () { ID=$1 INDICES=$2 - EXISTS=$(curl -s $KIBANA_URL/api/data_views/data_view/$ID -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H 'elastic-api-version: 2023-10-31') + EXISTS=$(curl -s $KIBANA_URL/api/data_views/data_view/$ID -u "$KIBANA_USER:$KIBANA_PASSWORD" -H 'elastic-api-version: 2023-10-31') + + title "Attempting to create data view $ID" if [[ $EXISTS == '{"statusCode":404,"error":"Not Found"'* ]]; then title "Creating data view $1 for indices $2" - curl -X POST $KIBANA_URL/api/data_views/data_view -H 'kbn-xsrf: bananas' -H 'Content-Type: application/json' -H 'elastic-api-version: 2023-10-31' -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -d "{\"data_view\": { \"name\": \"$1 (Automated by Demo CLI)\", \"title\": \"$2\", \"id\": \"$1\", \"timeFieldName\": \"@timestamp\" }}" + curl -X POST $KIBANA_URL/api/data_views/data_view -u "$KIBANA_USER:$KIBANA_PASSWORD" -H 'kbn-xsrf: bananas' -H 'Content-Type: application/json' -H 'elastic-api-version: 2023-10-31' -d "{\"data_view\": { \"name\": \"$1 (Automated by Demo CLI)\", \"title\": \"$2\", \"id\": \"$1\", \"timeFieldName\": \"@timestamp\" }}" else title "Data view $1 already exists" fi @@ -24,8 +33,10 @@ create_data_view () { create_rule () { ID=$1 DATA_FILE=$2 + + title "Attempting to create rule from $DATA_FILE" - curl -X POST $KIBANA_URL/api/alerting/rule/$ID -H 'kbn-xsrf: bananas' -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H 'Content-Type: application/json; elastic-api-version: 2023-10-31' -d @$SCRIPTS_DIR/data/$DATA_FILE + curl -X POST $KIBANA_URL/api/alerting/rule/$ID -u "$KIBANA_USER:$KIBANA_PASSWORD" -H 'kbn-xsrf: bananas' -H 'Content-Type: application/json; elastic-api-version: 2023-10-31' -d @$SCRIPTS_DIR/data/$DATA_FILE } title "Setting up Kibana assets for the demo..." @@ -34,12 +45,27 @@ if [ -z "${KIBANA_URL}" ]; then die "You must set KIBANA_URL so that we can post API requests there (please include base path if one exists)" fi +if [ -z "${OTEL_DEMO_ES_ENDPOINT}" ]; then + die "You must set OTEL_DEMO_ES_ENDPOINT" +fi + if [ -z "${OTEL_DEMO_ES_API_KEY}" ]; then - die "You must set OTEL_DEMO_ES_API_KEY (must be valid API key for ES and Kibana APIs)" + die "You must set OTEL_DEMO_ES_API_KEY (must be valid API key for ES APIs)" fi +if [ -z "${KIBANA_USER}" ]; then + die "You must set KIBANA_USER to an admin user" +fi + +if [ -z "${KIBANA_PASSWORD}" ]; then + die "You must set KIBANA_PASSWORD for the specified user" +fi + +# Replace kubernetes internal host pattern with localhost +OTEL_DEMO_ES_ENDPOINT=$(echo $OTEL_DEMO_ES_ENDPOINT | sed "s/host.minikube.internal/localhost/" | sed "s/host.docker.internal/localhost/") + # Create data views -create_data_view otel_logs_data logs-*otel* +# create_data_view otel_logs_data logs-*otel* # Create rules create_rule 9055220c-8fb1-4f9f-be7c-0a33eb2bafc5 rule-01.json @@ -50,10 +76,10 @@ echo -e "\n" # NOTE: this will fail if the data stream is not created # TODO: Update the component template also, and just swallow errors on this call title "Attempting to add message alias for otel logs" -curl -X PUT http://localhost:9200/logs-*otel*/_mapping -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H "Content-Type: application/json" -d @$SCRIPTS_DIR/data/mapping-message-alias.json +curl -X PUT "$OTEL_DEMO_ES_ENDPOINT/logs-*otel*/_mapping" -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H "Content-Type: application/json" -d @$SCRIPTS_DIR/data/mapping-message-alias.json echo -e "\n" # Enable EEM title "Attempting to enable EEM" -curl -X PUT -H "Authorization: ApiKey $OTEL_DEMO_ES_API_KEY" -H "kbn-xsrf: bananas" $KIBANA_URL/internal/entities/managed/enablement \ No newline at end of file +curl -X PUT -u "$KIBANA_USER:$KIBANA_PASSWORD" -H "kbn-xsrf: bananas" -H "x-elastic-internal-origin: Kibana" $KIBANA_URL/internal/entities/managed/enablement \ No newline at end of file diff --git a/scripts/rca-demo/setup b/scripts/rca-demo/setup index d7e09aafdc..9de2dfe8a8 100755 --- a/scripts/rca-demo/setup +++ b/scripts/rca-demo/setup @@ -10,12 +10,16 @@ set +a title "Checking environment for $APP_NAME..." # Load local user .env file -ENV_FILE="./scripts/rca-demo/.env" -if [ -f $ENV_FILE ]; then - echo "Sourcing env vars from $ENV_FILE ..." - source $ENV_FILE +LOAD_ENV_FILE_PATH="./scripts/rca-demo/.env" + +if [ -n "${ENV_FILE_PATH}" ]; then + LOAD_ENV_FILE_PATH=$ENV_FILE_PATH fi +if [ -f $LOAD_ENV_FILE_PATH ]; then + title "Sourcing environment variables from $LOAD_ENV_FILE_PATH" + source $LOAD_ENV_FILE_PATH +fi # Check tools tools=("helm" "kubectl" "minikube" "node" "npx") @@ -45,25 +49,6 @@ if [ -z "${OTEL_DEMO_ES_API_KEY}" ]; then die "You must set OTEL_DEMO_ES_API_KEY" fi -if [ -z "${SYNTHETICS_API_KEY}" ]; then - echo "No SYNTHETICS_API_KEY set, will skip synthetics setup" -else - echo "By providing a SYNTHETICS_API_KEY value, you are opting IN to synthetics setup" - echo "Please remove this env var or set it to \"\" to opt out." - - if [ -z "${KIBANA_URL}" ]; then - die "[Synthetics Requirement] You must set KIBANA_URL" - fi - - if [ -z "${OTEL_DEMO_FLEET_URL}" ]; then - die "[Synthetics Requirement] You must set OTEL_DEMO_FLEET_URL" - fi - - if [ -z "${OTEL_DEMO_FLEET_ENROLLMENT_TOKEN}" ]; then - die "[Synthetics Requirement] You must set OTEL_DEMO_FLEET_ENROLLMENT_TOKEN" - fi -fi - : ${TLS_INSECURE:="false"} if minikube status | grep -q "Running"; then @@ -148,19 +133,6 @@ pushd_helm_dir helminst nginx-ingress ./ingress popd -# If synthetics is configured -if [ -n "${SYNTHETICS_API_KEY}" ]; then - title "Installing synthetics agent helm chart" - pushd_helm_dir - helminst otel-synthetics-agent ./otel-synthetics - popd - - title "Pushing Synthetics project..." - cd synthetic-monitors - npm install - yes | npx @elastic/synthetics push . --params '{"url": "http://ingress-nginx-controller.ingress-nginx.svc.cluster.local"}' -fi - title "Setup complete!" cat <<-EOF You may access the demo at http://otel-demo.internal