From 288b8ae6eea201ef25c558e676cbaaaa7af4dd46 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 14:22:22 +0200 Subject: [PATCH 01/16] Add Elasticsearch example --- examples/loader-elasticsearch/.gitignore | 5 + examples/loader-elasticsearch/README.md | 9 + .../observablehq.config.js | 3 + examples/loader-elasticsearch/package.json | 26 +++ examples/loader-elasticsearch/src/.gitignore | 1 + .../src/data/es_client.ts | 63 ++++++ .../src/data/kibana_sample_data_logs.csv | 184 ++++++++++++++++++ .../src/data/kibana_sample_data_logs.csv.ts | 66 +++++++ examples/loader-elasticsearch/src/index.md | 182 +++++++++++++++++ 9 files changed, 539 insertions(+) create mode 100644 examples/loader-elasticsearch/.gitignore create mode 100644 examples/loader-elasticsearch/README.md create mode 100644 examples/loader-elasticsearch/observablehq.config.js create mode 100644 examples/loader-elasticsearch/package.json create mode 100644 examples/loader-elasticsearch/src/.gitignore create mode 100644 examples/loader-elasticsearch/src/data/es_client.ts create mode 100644 examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv create mode 100644 examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv.ts create mode 100644 examples/loader-elasticsearch/src/index.md diff --git a/examples/loader-elasticsearch/.gitignore b/examples/loader-elasticsearch/.gitignore new file mode 100644 index 000000000..0210d7df8 --- /dev/null +++ b/examples/loader-elasticsearch/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +.env +/dist/ +node_modules/ +yarn-error.log diff --git a/examples/loader-elasticsearch/README.md b/examples/loader-elasticsearch/README.md new file mode 100644 index 000000000..32fa60996 --- /dev/null +++ b/examples/loader-elasticsearch/README.md @@ -0,0 +1,9 @@ +[Framework examples →](../) + +# Elasticsearch data loader + +View live: + +This Observable Framework example demonstrates how to write a TypeScript data loader that runs a query on Elasticsearch using the [Elasticsearch Node.js client](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html). The data loader lives in [`src/data/kibana_sample_data_logs.csv.ts`](./src/data/kibana_logs_daily.csv.ts) and uses the helper [`src/data/es_client.ts`](./src/data/es_client.ts). + +To fully reproduce the example, you need to have a setup with both Elasticsearch and Kibana running to create the sample data. The dataset can be created from the UI via this URL: https://:/app/home#/tutorial_directory/sampleData diff --git a/examples/loader-elasticsearch/observablehq.config.js b/examples/loader-elasticsearch/observablehq.config.js new file mode 100644 index 000000000..fb0f92431 --- /dev/null +++ b/examples/loader-elasticsearch/observablehq.config.js @@ -0,0 +1,3 @@ +export default { + root: "src" +}; diff --git a/examples/loader-elasticsearch/package.json b/examples/loader-elasticsearch/package.json new file mode 100644 index 000000000..449648f01 --- /dev/null +++ b/examples/loader-elasticsearch/package.json @@ -0,0 +1,26 @@ +{ + "type": "module", + "private": true, + "scripts": { + "clean": "rimraf src/.observablehq/cache", + "build": "rimraf dist && observable build", + "dev": "observable preview", + "deploy": "observable deploy", + "observable": "observable" + }, + "dependencies": { + "@elastic/elasticsearch": "^8.14.0", + "@observablehq/framework": "^1.7.0", + "d3-dsv": "^3.0.1", + "d3-time": "^3.1.0", + "dotenv": "^16.4.5" + }, + "devDependencies": { + "@types/d3-dsv": "^3.0.7", + "@types/d3-time": "^3.0.3", + "rimraf": "^5.0.5" + }, + "engines": { + "node": ">=20" + } +} diff --git a/examples/loader-elasticsearch/src/.gitignore b/examples/loader-elasticsearch/src/.gitignore new file mode 100644 index 000000000..1235d15eb --- /dev/null +++ b/examples/loader-elasticsearch/src/.gitignore @@ -0,0 +1 @@ +/.observablehq/cache/ diff --git a/examples/loader-elasticsearch/src/data/es_client.ts b/examples/loader-elasticsearch/src/data/es_client.ts new file mode 100644 index 000000000..3144d1463 --- /dev/null +++ b/examples/loader-elasticsearch/src/data/es_client.ts @@ -0,0 +1,63 @@ +import "dotenv/config"; +import { Client } from "@elastic/elasticsearch"; + +// Have a look at the "Getting started" guide of the Elasticsearch node.js client +// to learn more about how to configure these environment variables: +// https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/getting-started-js.html + +const { + // ES_NODE can include the username and password in the URL, e.g.: + // ES_NODE=https://:@:9200 + ES_NODE, + // As an alternative to ES_NODE when using Elastic Cloud, you can use ES_CLOUD_ID and + // set it to the Cloud ID that you can find in the cloud console of the deployment (https://cloud.elastic.co/). + ES_CLOUD_ID, + // ES_API_KEY can be used instead of username and password. + // The API key will take precedence if both are set. + ES_API_KEY, + ES_USERNAME, + ES_PASSWORD, + // Warning: This option should be considered an insecure workaround for local development only. + // You may wish to specify a self-signed certificate rather than disabling certificate verification. + // ES_UNSAFE_TLS_REJECT_UNAUTHORIZED can be set to TRUE to disable certificate verification. + // See https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-connecting.html#auth-tls for more. + ES_UNSAFE_TLS_REJECT_UNAUTHORIZED, +} = process.env; + +if ((!ES_NODE && !ES_CLOUD_ID) || (ES_NODE && ES_CLOUD_ID)) + throw new Error( + "Either ES_NODE or ES_CLOUD_ID need to be defined, but not both.", + ); + +const esUrl = ES_NODE ? new URL(ES_NODE) : undefined; +const isHTTPS = esUrl?.protocol === "https:"; +const isLocalhost = esUrl?.hostname === "localhost"; + +export const esClient = new Client({ + ...(ES_NODE ? { node: ES_NODE } : {}), + ...(ES_CLOUD_ID ? { cloud: { id: ES_CLOUD_ID } } : {}), + ...(ES_API_KEY + ? { + auth: { + apiKey: ES_API_KEY, + }, + } + : {}), + ...(!ES_API_KEY && ES_USERNAME && ES_PASSWORD + ? { + auth: { + username: ES_USERNAME, + password: ES_PASSWORD, + }, + } + : {}), + ...(isHTTPS && + isLocalhost && + ES_UNSAFE_TLS_REJECT_UNAUTHORIZED?.toLowerCase() === "true" + ? { + tls: { + rejectUnauthorized: false, + }, + } + : {}), +}); diff --git a/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv b/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv new file mode 100644 index 000000000..aff60bffe --- /dev/null +++ b/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv @@ -0,0 +1,184 @@ +date,count,response_code +2024-06-16,230,200 +2024-06-16,12,503 +2024-06-16,7,404 +2024-06-17,209,200 +2024-06-17,12,404 +2024-06-17,10,503 +2024-06-18,210,200 +2024-06-18,16,404 +2024-06-18,4,503 +2024-06-19,220,200 +2024-06-19,11,404 +2024-06-19,5,503 +2024-06-20,211,200 +2024-06-20,15,404 +2024-06-20,4,503 +2024-06-21,210,200 +2024-06-21,12,404 +2024-06-21,8,503 +2024-06-22,211,200 +2024-06-22,10,503 +2024-06-22,8,404 +2024-06-23,219,200 +2024-06-23,9,404 +2024-06-23,3,503 +2024-06-24,211,200 +2024-06-24,12,404 +2024-06-24,7,503 +2024-06-25,208,200 +2024-06-25,11,404 +2024-06-25,11,503 +2024-06-26,222,200 +2024-06-26,7,503 +2024-06-26,1,404 +2024-06-27,209,200 +2024-06-27,13,503 +2024-06-27,8,404 +2024-06-28,203,200 +2024-06-28,18,404 +2024-06-28,9,503 +2024-06-29,216,200 +2024-06-29,8,404 +2024-06-29,6,503 +2024-06-30,214,200 +2024-06-30,10,404 +2024-06-30,6,503 +2024-07-01,212,200 +2024-07-01,12,404 +2024-07-01,6,503 +2024-07-02,215,200 +2024-07-02,12,404 +2024-07-02,3,503 +2024-07-03,212,200 +2024-07-03,11,404 +2024-07-03,6,503 +2024-07-04,210,200 +2024-07-04,14,404 +2024-07-04,7,503 +2024-07-05,211,200 +2024-07-05,14,404 +2024-07-05,5,503 +2024-07-06,196,200 +2024-07-06,32,404 +2024-07-06,2,503 +2024-07-07,205,200 +2024-07-07,17,404 +2024-07-07,8,503 +2024-07-08,204,200 +2024-07-08,14,404 +2024-07-08,12,503 +2024-07-09,202,200 +2024-07-09,17,404 +2024-07-09,11,503 +2024-07-10,214,200 +2024-07-10,10,503 +2024-07-10,6,404 +2024-07-11,206,200 +2024-07-11,13,404 +2024-07-11,11,503 +2024-07-12,218,200 +2024-07-12,6,404 +2024-07-12,6,503 +2024-07-13,216,200 +2024-07-13,10,404 +2024-07-13,4,503 +2024-07-14,216,200 +2024-07-14,7,404 +2024-07-14,6,503 +2024-07-15,218,200 +2024-07-15,12,404 +2024-07-15,1,503 +2024-07-16,162,200 +2024-07-16,7,404 +2024-07-16,4,503 +2024-07-17,211,200 +2024-07-17,14,404 +2024-07-17,5,503 +2024-07-18,214,200 +2024-07-18,9,404 +2024-07-18,7,503 +2024-07-19,213,200 +2024-07-19,9,404 +2024-07-19,8,503 +2024-07-20,212,200 +2024-07-20,10,404 +2024-07-20,7,503 +2024-07-21,203,200 +2024-07-21,21,404 +2024-07-21,7,503 +2024-07-22,215,200 +2024-07-22,8,503 +2024-07-22,7,404 +2024-07-23,208,200 +2024-07-23,17,404 +2024-07-23,4,503 +2024-07-24,212,200 +2024-07-24,13,404 +2024-07-24,6,503 +2024-07-25,211,200 +2024-07-25,13,404 +2024-07-25,6,503 +2024-07-26,213,200 +2024-07-26,9,404 +2024-07-26,8,503 +2024-07-27,210,200 +2024-07-27,110,404 +2024-07-27,9,503 +2024-07-28,217,200 +2024-07-28,8,404 +2024-07-28,6,503 +2024-07-29,200,200 +2024-07-29,17,404 +2024-07-29,13,503 +2024-07-30,215,200 +2024-07-30,12,404 +2024-07-30,3,503 +2024-07-31,212,200 +2024-07-31,11,404 +2024-07-31,7,503 +2024-08-01,206,200 +2024-08-01,13,503 +2024-08-01,11,404 +2024-08-02,216,200 +2024-08-02,7,404 +2024-08-02,7,503 +2024-08-03,214,200 +2024-08-03,14,404 +2024-08-03,2,503 +2024-08-04,213,200 +2024-08-04,13,503 +2024-08-04,4,404 +2024-08-05,212,200 +2024-08-05,9,404 +2024-08-05,9,503 +2024-08-06,211,200 +2024-08-06,11,503 +2024-08-06,8,404 +2024-08-07,210,200 +2024-08-07,13,404 +2024-08-07,7,503 +2024-08-08,206,200 +2024-08-08,16,404 +2024-08-08,8,503 +2024-08-09,211,200 +2024-08-09,13,404 +2024-08-09,6,503 +2024-08-10,212,200 +2024-08-10,13,404 +2024-08-10,5,503 +2024-08-11,208,200 +2024-08-11,12,404 +2024-08-11,10,503 +2024-08-12,210,200 +2024-08-12,10,404 +2024-08-12,10,503 +2024-08-13,213,200 +2024-08-13,9,503 +2024-08-13,8,404 +2024-08-14,210,200 +2024-08-14,13,404 +2024-08-14,7,503 +2024-08-15,194,200 +2024-08-15,8,404 +2024-08-15,3,503 diff --git a/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv.ts b/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv.ts new file mode 100644 index 000000000..c2b4dad1d --- /dev/null +++ b/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv.ts @@ -0,0 +1,66 @@ +import { csvFormat } from "d3-dsv"; +import { esClient } from "./es_client.js"; + +interface AggsResponseFormat { + logs_histogram: { + buckets: Array<{ + key: number; + key_as_string: string; + doc_count: number; + response_code: { + buckets: Array<{ key: string; doc_count: number }>; + }; + }>; + }; +} + +interface LoaderOutputFormat { + date: string; + count: number; + response_code: string; +} + +const resp = await esClient.search({ + index: "kibana_sample_data_logs", + size: 0, + aggs: { + logs_histogram: { + date_histogram: { + field: "@timestamp", + calendar_interval: "1d", + }, + aggs: { + response_code: { + terms: { + field: "response.keyword", + }, + }, + }, + }, + }, +}); + +if (!resp.aggregations) { + throw new Error("aggregations not defined"); +} + +process.stdout.write( + csvFormat( + // This transforms the nested response from Elasticsearch into a flat array. + resp.aggregations.logs_histogram.buckets.reduce>( + (p, c) => { + p.push( + ...c.response_code.buckets.map((d) => ({ + // Just keep the date from the full ISO string. + date: c.key_as_string.split("T")[0], + count: d.doc_count, + response_code: d.key, + })), + ); + + return p; + }, + [], + ), + ), +); diff --git a/examples/loader-elasticsearch/src/index.md b/examples/loader-elasticsearch/src/index.md new file mode 100644 index 000000000..f0952ee10 --- /dev/null +++ b/examples/loader-elasticsearch/src/index.md @@ -0,0 +1,182 @@ +# PostgreSQL data loader + +# Elasticsearch data loader + +Here’s a TypeScript data loader that queries an Elasticsearch cluster. + +```ts +import { csvFormat } from "d3-dsv"; +import { esClient } from "./es_client.js"; + +interface AggsResponseFormat { + logs_histogram: { + buckets: Array<{ + key: number; + key_as_string: string; + doc_count: number; + response_code: { + buckets: Array<{ key: string; doc_count: number }>; + }; + }>; + }; +} + +interface LoaderOutputFormat { + date: string; + count: number; + response_code: string; +} + +const resp = await esClient.search({ + index: "kibana_sample_data_logs", + size: 0, + aggs: { + logs_histogram: { + date_histogram: { + field: "@timestamp", + calendar_interval: "1d", + }, + aggs: { + response_code: { + terms: { + field: "response.keyword", + }, + }, + }, + }, + }, +}); + +if (!resp.aggregations) { + throw new Error("aggregations not defined"); +} + +process.stdout.write( + csvFormat( + // This transforms the nested response from Elasticsearch into a flat array. + resp.aggregations.logs_histogram.buckets.reduce>( + (p, c) => { + p.push( + ...c.response_code.buckets.map((d) => ({ + date: c.key_as_string, + count: d.doc_count, + response_code: d.key, + })), + ); + + return p; + }, + [], + ), + ), +); + +``` + +The data loader uses a helper file, `es_client.ts`, which provides a wrapper on the `@elastic/elasticsearch` package. This reduces the amount of boilerplate you need to write to issue a query. + +```ts +import "dotenv/config"; +import { Client } from "@elastic/elasticsearch"; + +// Have a look at the "Getting started" guide of the Elasticsearch node.js client +// to learn how to configure these environment variables: +// https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/getting-started-js.html + +const { + // ES_NODE can include the username and password in the URL, e.g.: + // ES_NODE=https://:@:9200 + ES_NODE, + // As an alternative to ES_NODE when using Elastic Cloud, you can use ES_CLOUD_ID and + // set it to the Cloud ID that you can find in the cloud console of the deployment (https://cloud.elastic.co/). + ES_CLOUD_ID, + // ES_API_KEY can be used instead of username and password. + // The API key will take precedence if both are set. + ES_API_KEY, + ES_USERNAME, + ES_PASSWORD, + // Warning: This option should be considered an insecure workaround for local development only. + // You may wish to specify a self-signed certificate rather than disabling certificate verification. + // ES_UNSAFE_TLS_REJECT_UNAUTHORIZED can be set to TRUE to disable certificate verification. + // See https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-connecting.html#auth-tls for more. + ES_UNSAFE_TLS_REJECT_UNAUTHORIZED, +} = process.env; + +if ((!ES_NODE && !ES_CLOUD_ID) || (ES_NODE && ES_CLOUD_ID)) + throw new Error( + "Either ES_NODE or ES_CLOUD_ID need to be defined, but not both.", + ); + +const esUrl = ES_NODE ? new URL(ES_NODE) : undefined; +const isHTTPS = esUrl?.protocol === "https:"; +const isLocalhost = esUrl?.hostname === "localhost"; + +export const esClient = new Client({ + ...(ES_NODE ? { node: ES_NODE } : {}), + ...(ES_CLOUD_ID ? { cloud: { id: ES_CLOUD_ID } } : {}), + ...(ES_API_KEY + ? { + auth: { + apiKey: ES_API_KEY, + }, + } + : {}), + ...(!ES_API_KEY && ES_USERNAME && ES_PASSWORD + ? { + auth: { + username: ES_USERNAME, + password: ES_PASSWORD, + }, + } + : {}), + ...(isHTTPS && + isLocalhost && + ES_UNSAFE_TLS_REJECT_UNAUTHORIZED?.toLowerCase() === "true" + ? { + tls: { + rejectUnauthorized: false, + }, + } + : {}), +}); +``` + +
+To run this data loader, you’ll need to install `@elastic/elasticsearch`, `d3-dsv` and `dotenv` using your preferred package manager such as npm or Yarn. +
+ +For the data loader to authenticate with your Elasticsearch cluster, you need to set the environment variables defined in the helper. If you use GitHub, you can use [secrets in GitHub Actions](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) to set environment variables; other platforms provide similar functionality for continuous deployment. For local development, we use the `dotenv` package, which allows environment variables to be defined in a `.env` file which lives in the project root and looks like this: + +``` +ES_NODE="https://USERNAME:PASSWORD@HOST:9200" +``` + +
+The `.env` file should not be committed to your source code repository; keep your credentials secret. +
+ +The above data loader lives in `data/kibana_sample_data_logs.csv.ts`, so we can load the data as `data/kibana_sample_data_logs.csv`. The `FileAttachment.csv` method parses the file and returns a promise to an array of objects. + +```js echo +const logs = FileAttachment("./data/kibana_sample_data_logs.csv").csv({typed: true}); +``` + +The `logs` table has three columns: `date`, `count` and `response_code`. We can display the table using `Inputs.table`. + +```js echo +Inputs.table(logs) +``` + +Lastly, we can pass the table to `Plot.plot` to make a line chart. + +```js echo +Plot.plot({ + style: "overflow: visible;", + y: { grid: true }, + marks: [ + Plot.ruleY([0]), + Plot.line(logs, {x: "date", y: "count", stroke: "response_code", tip: true}), + Plot.text(logs, Plot.selectLast({x: "date", y: "count", z: "response_code", text: "response_code", textAnchor: "start", dx: 3})) + ] +}) +``` From 9302ad3000eabe648f82dc022ee73bd8eab83a69 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 14:25:50 +0200 Subject: [PATCH 02/16] Fix links --- examples/loader-elasticsearch/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/loader-elasticsearch/README.md b/examples/loader-elasticsearch/README.md index 32fa60996..872fbbb81 100644 --- a/examples/loader-elasticsearch/README.md +++ b/examples/loader-elasticsearch/README.md @@ -2,8 +2,8 @@ # Elasticsearch data loader -View live: +View live: -This Observable Framework example demonstrates how to write a TypeScript data loader that runs a query on Elasticsearch using the [Elasticsearch Node.js client](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html). The data loader lives in [`src/data/kibana_sample_data_logs.csv.ts`](./src/data/kibana_logs_daily.csv.ts) and uses the helper [`src/data/es_client.ts`](./src/data/es_client.ts). +This Observable Framework example demonstrates how to write a TypeScript data loader that runs a query on Elasticsearch using the [Elasticsearch Node.js client](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html). The data loader lives in [`src/data/kibana_sample_data_logs.csv.ts`](./src/data/kibana_sample_data_logs.csv.ts) and uses the helper [`src/data/es_client.ts`](./src/data/es_client.ts). To fully reproduce the example, you need to have a setup with both Elasticsearch and Kibana running to create the sample data. The dataset can be created from the UI via this URL: https://:/app/home#/tutorial_directory/sampleData From 349207d22467c2c48b71aa8935c60b3c7d7427c2 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:14:16 +0200 Subject: [PATCH 03/16] adds info how to set up elasticsearch/kibana --- examples/loader-elasticsearch/README.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/examples/loader-elasticsearch/README.md b/examples/loader-elasticsearch/README.md index 872fbbb81..c637318db 100644 --- a/examples/loader-elasticsearch/README.md +++ b/examples/loader-elasticsearch/README.md @@ -6,4 +6,26 @@ View live: :/app/home#/tutorial_directory/sampleData +To fully reproduce the example, you need to have a setup with both Elasticsearch and Kibana running to create the sample data. Here's how to set up both on macOS: + +```bash +# Download and run Elasticsearch +curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.14.1-darwin-x86_64.tar.gz +gunzip -c elasticsearch-8.14.1-darwin-x86_64.tar.gz | tar xopf - +cd elasticsearch-8.14.1 +./bin/elasticsearch + +# Next, in another terminal tab, download and run Kibana +curl -O https://artifacts.elastic.co/downloads/kibana/kibana-8.14.1-darwin-x86_64.tar.gz +gunzip -c kibana-8.14.1-darwin-x86_64.tar.gz | tar xopf - +cd kibana-8.14.1 +./bin/kibana +``` + +The commands for both will output instructions how to finish the setup with security enabled. Once you have both running, you can create the sample data in Kibana. via this URL: http://localhost:5601/app/home#/tutorial_directory/sampleData + +Finally, create the `.env` file with the credentials shared for the user `elastic` that were logged when starting Elasticsearch like this: + +``` +ES_NODE=https://elastic:@localhost:9200 +``` From bccb7334ddd38ff11ead04e5f1acad290653cd89 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:15:55 +0200 Subject: [PATCH 04/16] fix README.md --- examples/loader-elasticsearch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/loader-elasticsearch/README.md b/examples/loader-elasticsearch/README.md index c637318db..235d1d5f5 100644 --- a/examples/loader-elasticsearch/README.md +++ b/examples/loader-elasticsearch/README.md @@ -22,7 +22,7 @@ cd kibana-8.14.1 ./bin/kibana ``` -The commands for both will output instructions how to finish the setup with security enabled. Once you have both running, you can create the sample data in Kibana. via this URL: http://localhost:5601/app/home#/tutorial_directory/sampleData +The commands for both will output instructions how to finish the setup with security enabled. Once you have both running, you can create the sample data in Kibana via this URL: http://localhost:5601/app/home#/tutorial_directory/sampleData Finally, create the `.env` file with the credentials shared for the user `elastic` that were logged when starting Elasticsearch like this: From 946d25f26d467a58df55f0ff349bfec7cb9fbd72 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:35:57 +0200 Subject: [PATCH 05/16] fix instructions to include NODE_CA_FINGERPRINT --- examples/loader-elasticsearch/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/loader-elasticsearch/README.md b/examples/loader-elasticsearch/README.md index 235d1d5f5..8a00a5431 100644 --- a/examples/loader-elasticsearch/README.md +++ b/examples/loader-elasticsearch/README.md @@ -24,8 +24,14 @@ cd kibana-8.14.1 The commands for both will output instructions how to finish the setup with security enabled. Once you have both running, you can create the sample data in Kibana via this URL: http://localhost:5601/app/home#/tutorial_directory/sampleData -Finally, create the `.env` file with the credentials shared for the user `elastic` that were logged when starting Elasticsearch like this: +Finally, create the `.env` file with the credentials shared for the user `elastic` that were logged when starting Elasticsearch like this. To get the CA fingerprint for the config, run the following command from the directory you started installing Elasticsearch: + +``` +openssl x509 -fingerprint -sha256 -noout -in ./elasticsearch-8.14.1/config/certs/http_ca.crt +``` ``` ES_NODE=https://elastic:@localhost:9200 +ES_CA_FINGERPRINT= +ES_UNSAFE_TLS_REJECT_UNAUTHORIZED=FALSE ``` From 7eaea8ecc5fc29a93df7669d249c7287f289a211 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:37:32 +0200 Subject: [PATCH 06/16] adds support for NODE_CA_FINGERPRINT to es_client.ts --- examples/loader-elasticsearch/src/data/es_client.ts | 6 +++++- examples/loader-elasticsearch/src/index.md | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/loader-elasticsearch/src/data/es_client.ts b/examples/loader-elasticsearch/src/data/es_client.ts index 3144d1463..7d9d16634 100644 --- a/examples/loader-elasticsearch/src/data/es_client.ts +++ b/examples/loader-elasticsearch/src/data/es_client.ts @@ -17,6 +17,9 @@ const { ES_API_KEY, ES_USERNAME, ES_PASSWORD, + // the fingerprint (SHA256) of the CA certificate that is used to sign + // the certificate that the Elasticsearch node presents for TLS. + ES_CA_FINGERPRINT, // Warning: This option should be considered an insecure workaround for local development only. // You may wish to specify a self-signed certificate rather than disabling certificate verification. // ES_UNSAFE_TLS_REJECT_UNAUTHORIZED can be set to TRUE to disable certificate verification. @@ -36,6 +39,7 @@ const isLocalhost = esUrl?.hostname === "localhost"; export const esClient = new Client({ ...(ES_NODE ? { node: ES_NODE } : {}), ...(ES_CLOUD_ID ? { cloud: { id: ES_CLOUD_ID } } : {}), + ...(ES_CA_FINGERPRINT ? { caFingerprint: ES_CA_FINGERPRINT } : {}), ...(ES_API_KEY ? { auth: { @@ -53,7 +57,7 @@ export const esClient = new Client({ : {}), ...(isHTTPS && isLocalhost && - ES_UNSAFE_TLS_REJECT_UNAUTHORIZED?.toLowerCase() === "true" + ES_UNSAFE_TLS_REJECT_UNAUTHORIZED?.toLowerCase() === "false" ? { tls: { rejectUnauthorized: false, diff --git a/examples/loader-elasticsearch/src/index.md b/examples/loader-elasticsearch/src/index.md index f0952ee10..c05e12bd0 100644 --- a/examples/loader-elasticsearch/src/index.md +++ b/examples/loader-elasticsearch/src/index.md @@ -80,7 +80,7 @@ import "dotenv/config"; import { Client } from "@elastic/elasticsearch"; // Have a look at the "Getting started" guide of the Elasticsearch node.js client -// to learn how to configure these environment variables: +// to learn more about how to configure these environment variables: // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/getting-started-js.html const { @@ -95,6 +95,9 @@ const { ES_API_KEY, ES_USERNAME, ES_PASSWORD, + // the fingerprint (SHA256) of the CA certificate that is used to sign + // the certificate that the Elasticsearch node presents for TLS. + ES_CA_FINGERPRINT, // Warning: This option should be considered an insecure workaround for local development only. // You may wish to specify a self-signed certificate rather than disabling certificate verification. // ES_UNSAFE_TLS_REJECT_UNAUTHORIZED can be set to TRUE to disable certificate verification. @@ -114,6 +117,7 @@ const isLocalhost = esUrl?.hostname === "localhost"; export const esClient = new Client({ ...(ES_NODE ? { node: ES_NODE } : {}), ...(ES_CLOUD_ID ? { cloud: { id: ES_CLOUD_ID } } : {}), + ...(ES_CA_FINGERPRINT ? { caFingerprint: ES_CA_FINGERPRINT } : {}), ...(ES_API_KEY ? { auth: { @@ -131,7 +135,7 @@ export const esClient = new Client({ : {}), ...(isHTTPS && isLocalhost && - ES_UNSAFE_TLS_REJECT_UNAUTHORIZED?.toLowerCase() === "true" + ES_UNSAFE_TLS_REJECT_UNAUTHORIZED?.toLowerCase() === "false" ? { tls: { rejectUnauthorized: false, From 07c1d588e471a9bdcba1ab706406c588fc7f10b6 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:48:19 +0200 Subject: [PATCH 07/16] fix comment about ES_UNSAFE_TLS_REJECT_UNAUTHORIZED --- examples/loader-elasticsearch/src/data/es_client.ts | 2 +- examples/loader-elasticsearch/src/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/loader-elasticsearch/src/data/es_client.ts b/examples/loader-elasticsearch/src/data/es_client.ts index 7d9d16634..23ccd3bd4 100644 --- a/examples/loader-elasticsearch/src/data/es_client.ts +++ b/examples/loader-elasticsearch/src/data/es_client.ts @@ -22,7 +22,7 @@ const { ES_CA_FINGERPRINT, // Warning: This option should be considered an insecure workaround for local development only. // You may wish to specify a self-signed certificate rather than disabling certificate verification. - // ES_UNSAFE_TLS_REJECT_UNAUTHORIZED can be set to TRUE to disable certificate verification. + // ES_UNSAFE_TLS_REJECT_UNAUTHORIZED can be set to FALSE to disable certificate verification. // See https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-connecting.html#auth-tls for more. ES_UNSAFE_TLS_REJECT_UNAUTHORIZED, } = process.env; diff --git a/examples/loader-elasticsearch/src/index.md b/examples/loader-elasticsearch/src/index.md index c05e12bd0..9e00a77f1 100644 --- a/examples/loader-elasticsearch/src/index.md +++ b/examples/loader-elasticsearch/src/index.md @@ -100,7 +100,7 @@ const { ES_CA_FINGERPRINT, // Warning: This option should be considered an insecure workaround for local development only. // You may wish to specify a self-signed certificate rather than disabling certificate verification. - // ES_UNSAFE_TLS_REJECT_UNAUTHORIZED can be set to TRUE to disable certificate verification. + // ES_UNSAFE_TLS_REJECT_UNAUTHORIZED can be set to FALSE to disable certificate verification. // See https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-connecting.html#auth-tls for more. ES_UNSAFE_TLS_REJECT_UNAUTHORIZED, } = process.env; From 9402e08c56d2293b632508064bb839fdc6917d2c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:52:18 +0200 Subject: [PATCH 08/16] tweak loader --- .../src/data/kibana_sample_data_logs.csv.ts | 6 +----- examples/loader-elasticsearch/src/index.md | 10 +++------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv.ts b/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv.ts index c2b4dad1d..1ff13918e 100644 --- a/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv.ts +++ b/examples/loader-elasticsearch/src/data/kibana_sample_data_logs.csv.ts @@ -40,14 +40,10 @@ const resp = await esClient.search({ }, }); -if (!resp.aggregations) { - throw new Error("aggregations not defined"); -} - process.stdout.write( csvFormat( // This transforms the nested response from Elasticsearch into a flat array. - resp.aggregations.logs_histogram.buckets.reduce>( + resp.aggregations!.logs_histogram.buckets.reduce>( (p, c) => { p.push( ...c.response_code.buckets.map((d) => ({ diff --git a/examples/loader-elasticsearch/src/index.md b/examples/loader-elasticsearch/src/index.md index 9e00a77f1..09405da32 100644 --- a/examples/loader-elasticsearch/src/index.md +++ b/examples/loader-elasticsearch/src/index.md @@ -47,18 +47,15 @@ const resp = await esClient.search({ }, }); -if (!resp.aggregations) { - throw new Error("aggregations not defined"); -} - process.stdout.write( csvFormat( // This transforms the nested response from Elasticsearch into a flat array. - resp.aggregations.logs_histogram.buckets.reduce>( + resp.aggregations!.logs_histogram.buckets.reduce>( (p, c) => { p.push( ...c.response_code.buckets.map((d) => ({ - date: c.key_as_string, + // Just keep the date from the full ISO string. + date: c.key_as_string.split("T")[0], count: d.doc_count, response_code: d.key, })), @@ -70,7 +67,6 @@ process.stdout.write( ), ), ); - ``` The data loader uses a helper file, `es_client.ts`, which provides a wrapper on the `@elastic/elasticsearch` package. This reduces the amount of boilerplate you need to write to issue a query. From 9e64b1a254e7fff29c6fe36d103958c7caade48a Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:53:07 +0200 Subject: [PATCH 09/16] remove postgres title --- examples/loader-elasticsearch/src/index.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/loader-elasticsearch/src/index.md b/examples/loader-elasticsearch/src/index.md index 09405da32..376d4b07b 100644 --- a/examples/loader-elasticsearch/src/index.md +++ b/examples/loader-elasticsearch/src/index.md @@ -1,5 +1,3 @@ -# PostgreSQL data loader - # Elasticsearch data loader Here’s a TypeScript data loader that queries an Elasticsearch cluster. From 77bd12d14170d641f42a906c1a58839bb278850e Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:54:38 +0200 Subject: [PATCH 10/16] Update examples/loader-elasticsearch/src/index.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tweak text Co-authored-by: Philippe Rivière --- examples/loader-elasticsearch/src/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/loader-elasticsearch/src/index.md b/examples/loader-elasticsearch/src/index.md index 376d4b07b..9162c796b 100644 --- a/examples/loader-elasticsearch/src/index.md +++ b/examples/loader-elasticsearch/src/index.md @@ -165,7 +165,7 @@ The `logs` table has three columns: `date`, `count` and `response_code`. We can Inputs.table(logs) ``` -Lastly, we can pass the table to `Plot.plot` to make a line chart. +Lastly, we can pass the table to Observable Plot to make a line chart. ```js echo Plot.plot({ From 8a09785cb9f17c3b86f6150e526b9e176919b0c1 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:54:54 +0200 Subject: [PATCH 11/16] Update examples/loader-elasticsearch/src/index.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit adding empty lines should allow framework to convert the markdown (in this case, the backticks around .env) Co-authored-by: Philippe Rivière --- examples/loader-elasticsearch/src/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/loader-elasticsearch/src/index.md b/examples/loader-elasticsearch/src/index.md index 9162c796b..bd96ef524 100644 --- a/examples/loader-elasticsearch/src/index.md +++ b/examples/loader-elasticsearch/src/index.md @@ -150,7 +150,9 @@ ES_NODE="https://USERNAME:PASSWORD@HOST:9200" ```
+ The `.env` file should not be committed to your source code repository; keep your credentials secret. +
The above data loader lives in `data/kibana_sample_data_logs.csv.ts`, so we can load the data as `data/kibana_sample_data_logs.csv`. The `FileAttachment.csv` method parses the file and returns a promise to an array of objects. From f73002d0727c789c03f5cfda712d8fc9593fc86c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 15:55:06 +0200 Subject: [PATCH 12/16] Update examples/loader-elasticsearch/src/index.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit adding empty lines should allow framework to convert the markdown (in this case, the backticks around .env) Co-authored-by: Philippe Rivière --- examples/loader-elasticsearch/src/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/loader-elasticsearch/src/index.md b/examples/loader-elasticsearch/src/index.md index bd96ef524..4740f03d6 100644 --- a/examples/loader-elasticsearch/src/index.md +++ b/examples/loader-elasticsearch/src/index.md @@ -140,7 +140,9 @@ export const esClient = new Client({ ```
+ To run this data loader, you’ll need to install `@elastic/elasticsearch`, `d3-dsv` and `dotenv` using your preferred package manager such as npm or Yarn. +
For the data loader to authenticate with your Elasticsearch cluster, you need to set the environment variables defined in the helper. If you use GitHub, you can use [secrets in GitHub Actions](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) to set environment variables; other platforms provide similar functionality for continuous deployment. For local development, we use the `dotenv` package, which allows environment variables to be defined in a `.env` file which lives in the project root and looks like this: From aebd9108ecac1a97ab55f5588ffb04bb0f55d069 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 16:07:02 +0200 Subject: [PATCH 13/16] add link to examples/README.md --- examples/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/README.md b/examples/README.md index 531a24bd3..b39311b61 100644 --- a/examples/README.md +++ b/examples/README.md @@ -56,6 +56,7 @@ - [`loader-arrow`](https://observablehq.observablehq.cloud/framework-example-loader-arrow/) - Generating Apache Arrow IPC files - [`loader-databricks`](https://observablehq.observablehq.cloud/framework-example-loader-databricks/) - Loading data from Databricks - [`loader-duckdb`](https://observablehq.observablehq.cloud/framework-example-loader-duckdb/) - Processing data with DuckDB +- [`loader-elasticsearch`](https://observablehq.observablehq.cloud/framework-example-loader-elasticsearch/) - Loading data from Elasticsearch - [`loader-github`](https://observablehq.observablehq.cloud/framework-example-loader-github/) - Loading data from GitHub - [`loader-google-analytics`](https://observablehq.observablehq.cloud/framework-example-loader-google-analytics/) - Loading data from Google Analytics - [`loader-julia-to-txt`](https://observablehq.observablehq.cloud/framework-example-loader-julia-to-txt/) - Generating TXT from Julia From b2b4d0667caa5c8a5763ce176919f6ecec0ad9ff Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 27 Jun 2024 17:06:15 +0200 Subject: [PATCH 14/16] PR feedback: update obshq; mention dataset --- examples/loader-elasticsearch/README.md | 8 ++++---- examples/loader-elasticsearch/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/loader-elasticsearch/README.md b/examples/loader-elasticsearch/README.md index 8a00a5431..bc9a30907 100644 --- a/examples/loader-elasticsearch/README.md +++ b/examples/loader-elasticsearch/README.md @@ -22,7 +22,7 @@ cd kibana-8.14.1 ./bin/kibana ``` -The commands for both will output instructions how to finish the setup with security enabled. Once you have both running, you can create the sample data in Kibana via this URL: http://localhost:5601/app/home#/tutorial_directory/sampleData +The commands for both will output instructions how to finish the setup with security enabled. Once you have both running, you can create the "Sample web logs" dataset in Kibana via this URL: http://localhost:5601/app/home#/tutorial_directory/sampleData Finally, create the `.env` file with the credentials shared for the user `elastic` that were logged when starting Elasticsearch like this. To get the CA fingerprint for the config, run the following command from the directory you started installing Elasticsearch: @@ -31,7 +31,7 @@ openssl x509 -fingerprint -sha256 -noout -in ./elasticsearch-8.14.1/config/certs ``` ``` -ES_NODE=https://elastic:@localhost:9200 -ES_CA_FINGERPRINT= -ES_UNSAFE_TLS_REJECT_UNAUTHORIZED=FALSE +ES_NODE="https://elastic:@localhost:9200" +ES_CA_FINGERPRINT="" +ES_UNSAFE_TLS_REJECT_UNAUTHORIZED="FALSE" ``` diff --git a/examples/loader-elasticsearch/package.json b/examples/loader-elasticsearch/package.json index 449648f01..e7e554ef1 100644 --- a/examples/loader-elasticsearch/package.json +++ b/examples/loader-elasticsearch/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@elastic/elasticsearch": "^8.14.0", - "@observablehq/framework": "^1.7.0", + "@observablehq/framework": "latest", "d3-dsv": "^3.0.1", "d3-time": "^3.1.0", "dotenv": "^16.4.5" From 7d930058194d00151d4bdfc652661abe75f88609 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 28 Jun 2024 11:20:30 +0200 Subject: [PATCH 15/16] Update examples/loader-elasticsearch/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix quote Co-authored-by: Philippe Rivière --- examples/loader-elasticsearch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/loader-elasticsearch/README.md b/examples/loader-elasticsearch/README.md index bc9a30907..e6f0aac5e 100644 --- a/examples/loader-elasticsearch/README.md +++ b/examples/loader-elasticsearch/README.md @@ -6,7 +6,7 @@ View live: Date: Fri, 28 Jun 2024 11:20:40 +0200 Subject: [PATCH 16/16] Update examples/loader-elasticsearch/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix quotes Co-authored-by: Philippe Rivière --- examples/loader-elasticsearch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/loader-elasticsearch/README.md b/examples/loader-elasticsearch/README.md index e6f0aac5e..32b6ad124 100644 --- a/examples/loader-elasticsearch/README.md +++ b/examples/loader-elasticsearch/README.md @@ -22,7 +22,7 @@ cd kibana-8.14.1 ./bin/kibana ``` -The commands for both will output instructions how to finish the setup with security enabled. Once you have both running, you can create the "Sample web logs" dataset in Kibana via this URL: http://localhost:5601/app/home#/tutorial_directory/sampleData +The commands for both will output instructions how to finish the setup with security enabled. Once you have both running, you can create the “Sample web logs” dataset in Kibana via this URL: http://localhost:5601/app/home#/tutorial_directory/sampleData Finally, create the `.env` file with the credentials shared for the user `elastic` that were logged when starting Elasticsearch like this. To get the CA fingerprint for the config, run the following command from the directory you started installing Elasticsearch: