From 893aebc7d976e1e3ed459c63b0756ef35c4f70f7 Mon Sep 17 00:00:00 2001 From: Scott Lyons Date: Fri, 1 Nov 2024 13:06:15 -0700 Subject: [PATCH 1/4] Standardizing and enhancing Dremio documentation --- .../docs/components/data-connectors/dremio.md | 238 +++++++++--------- 1 file changed, 121 insertions(+), 117 deletions(-) diff --git a/spiceaidocs/docs/components/data-connectors/dremio.md b/spiceaidocs/docs/components/data-connectors/dremio.md index ddb4c7e2..d673a0ee 100644 --- a/spiceaidocs/docs/components/data-connectors/dremio.md +++ b/spiceaidocs/docs/components/data-connectors/dremio.md @@ -4,10 +4,9 @@ sidebar_label: 'Dremio Data Connector' description: 'Dremio Data Connector Documentation' --- -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; +[Dremio](https://www.dremio.com/) is a data lake engine that enables high-performance SQL queries directly on data lake storage. It provides a unified interface for querying and analyzing data from various sources without the need for complex data movement or transformation. -[Dremio](https://www.dremio.com/) server as a connector for federated SQL queries. +This connector enables using Dremio as a Data source for federated SQL queries. ```yaml - from: dremio:datasets.dremio_dataset @@ -20,117 +19,122 @@ import TabItem from '@theme/TabItem'; ## Configuration -- `dremio_endpoint`: The endpoint used to connect to the Dremio server. -- `dremio_username`: The username to connect with. -- `dremio_password`: The password to connect with. Use the [secret replacement syntax](../secret-stores/index.md) to load the password from a secret store, e.g. `${secrets:my_dremio_pass}`. - -## Auth Example - -Check [Secrets Stores](/components/secret-stores) for more details. - - - - - ```bash - SPICE_DREMIO_USERNAME=demo \ - SPICE_DREMIO_PASSWORD=demo1234 \ - spice run - # Or using the CLI to configure the secrets into an `.env` file - spice login dremio -u demo -p demo1234 - ``` - - `.env` - ```bash - SPICE_DREMIO_USERNAME=demo - SPICE_DREMIO_PASSWORD=demo1234 - ``` - - `spicepod.yaml` - ```yaml - version: v1beta1 - kind: Spicepod - name: spice-app - - secrets: - - from: env - name: env - - datasets: - - from: dremio:datasets.dremio_dataset - name: dremio_dataset - params: - dremio_endpoint: grpc://1.2.3.4:32010 - dremio_username: ${env:SPICE_DREMIO_USERNAME} - dremio_password: ${env:SPICE_DREMIO_PASSWORD} - ``` - - Learn more about [Env Secret Store](/components/secret-stores/env). - - - - - ```bash - kubectl create secret generic dremio \ - --from-literal=username='demo' \ - --from-literal=password='demo1234' - ``` - - `spicepod.yaml` - ```yaml - version: v1beta1 - kind: Spicepod - name: spice-app - - secrets: - - from: kubernetes:dremio - name: dremio - - datasets: - - from: dremio:datasets.dremio_dataset - name: dremio_dataset - params: - dremio_endpoint: grpc://1.2.3.4:32010 - dremio_username: ${dremio:username} - dremio_password: ${dremio:password} - ``` - - Learn more about [Kubernetes Secret Store](/components/secret-stores/kubernetes). - - - - Add new keychain entries (macOS) for the user and password: - - ```bash - # Add Username to keychain - security add-generic-password -l "Dremio Username" \ - -a spiced -s spice_dremio_username \ - -w demo - # Add Password to keychain - security add-generic-password -l "Dremio Password" \ - -a spiced -s spice_dremio_password \ - -w demo1234 - ``` - - `spicepod.yaml` - ```yaml - version: v1beta1 - kind: Spicepod - name: spice-app - - secrets: - - from: keyring - name: keyring - - datasets: - - from: dremio:datasets.dremio_dataset - name: dremio_dataset - params: - dremio_endpoint: grpc://1.2.3.4:32010 - dremio_username: ${keyring:spice_dremio_username} - dremio_password: ${keyring:spice_dremio_password} - ``` - - Learn more about [Keyring Secret Store](/components/secret-stores/keyring). - - - +### `from` + +The `from` field takes the form `dremio:dataset` where `dataset` is the fully qualified name of the dataset to read from. + +:::warning [Limitations] + +Only three nested levels of a dataset name is currently supported, e.g `a.b.c`. Any futher than that is not supported at this time. + +::: + +### `name` + +The dataset name. This will be used as the table name within Spice. + +Example: +```yaml +datasets: + - from: dremio:datasets.dremio_dataset + name: cool_dataset + params: + ... +``` + +```sql +SELECT COUNT(*) FROM cool_dataset; +``` + +```shell ++----------+ +| count(*) | ++----------+ +| 6001215 | ++----------+ +``` + +### `params` + +| Parameter Name | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `dremio_endpoint` | The endpoint used to connect to the Dremio server. | +| `dremio_username` | The username to connect with. | +| `dremio_password` | The password to connect with. Use the [secret replacement syntax](../secret-stores/index.md) to load the password from a secret store, e.g. `${secrets:my_dremio_pass}`. | + +## Examples + +### Environment Authentication + +connection parameters can be specified as inline variables: +```bash +SPICE_DREMIO_USERNAME=demo \ +SPICE_DREMIO_PASSWORD=demo1234 \ +spice run +``` + + +Or using the CLI to configure the secrets into an `.env` file +```bash +spice login dremio -u demo -p demo1234 +``` + +`.env` +```bash +SPICE_DREMIO_USERNAME=demo +SPICE_DREMIO_PASSWORD=demo1234 +``` + +Then configure the `spicepod.yaml`: +```yaml +version: v1beta1 +kind: Spicepod +name: spice-app + +secrets: + - from: env + name: env + +datasets: + - from: dremio:datasets.dremio_dataset + name: dremio_dataset + params: + dremio_endpoint: grpc://1.2.3.4:32010 + dremio_username: ${env:SPICE_DREMIO_USERNAME} + dremio_password: ${env:SPICE_DREMIO_PASSWORD} +``` + +### Kubernetes secrets + +```bash +kubectl create secret generic dremio \ + --from-literal=username='demo' \ + --from-literal=password='demo1234' +``` + +`spicepod.yaml` +```yaml +version: v1beta1 +kind: Spicepod +name: spice-app + +secrets: + - from: kubernetes:dremio + name: dremio + +datasets: + - from: dremio:datasets.dremio_dataset + name: dremio_dataset + params: + dremio_endpoint: grpc://1.2.3.4:32010 + dremio_username: ${dremio:username} + dremio_password: ${dremio:password} +``` + +## Using secrets + +There are currently three supported [secret stores](/components/secret-stores/index.md): + +* [Environment variables](/components/secret-stores/env) +* [Kubernetes Secret Store](/components/secret-stores/kubernetes) +* [Keyring Secret Store](/components/secret-stores/keyring) From 8741e2bb08113261642fb511b3ab10ea72ccfacd Mon Sep 17 00:00:00 2001 From: Scott Lyons Date: Sun, 10 Nov 2024 17:56:37 -0800 Subject: [PATCH 2/4] Updating secrets section --- .../docs/components/data-connectors/dremio.md | 88 +++---------------- 1 file changed, 14 insertions(+), 74 deletions(-) diff --git a/spiceaidocs/docs/components/data-connectors/dremio.md b/spiceaidocs/docs/components/data-connectors/dremio.md index d673a0ee..e92acf7b 100644 --- a/spiceaidocs/docs/components/data-connectors/dremio.md +++ b/spiceaidocs/docs/components/data-connectors/dremio.md @@ -56,85 +56,25 @@ SELECT COUNT(*) FROM cool_dataset; ### `params` -| Parameter Name | Description | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `dremio_endpoint` | The endpoint used to connect to the Dremio server. | -| `dremio_username` | The username to connect with. | -| `dremio_password` | The password to connect with. Use the [secret replacement syntax](../secret-stores/index.md) to load the password from a secret store, e.g. `${secrets:my_dremio_pass}`. | +| Parameter Name | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dremio_endpoint` | The endpoint used to connect to the Dremio server. | +| `dremio_username` | The username to connect with. | +| `dremio_password` | The password to connect with. Use the [secret replacement syntax](#secrets) to load the password from a secret store, e.g. `${secrets:my_dremio_pass}`. | ## Examples -### Environment Authentication +### Connecting to a GRPC endpoint -connection parameters can be specified as inline variables: -```bash -SPICE_DREMIO_USERNAME=demo \ -SPICE_DREMIO_PASSWORD=demo1234 \ -spice run -``` - - -Or using the CLI to configure the secrets into an `.env` file -```bash -spice login dremio -u demo -p demo1234 -``` - -`.env` -```bash -SPICE_DREMIO_USERNAME=demo -SPICE_DREMIO_PASSWORD=demo1234 -``` - -Then configure the `spicepod.yaml`: -```yaml -version: v1beta1 -kind: Spicepod -name: spice-app - -secrets: - - from: env - name: env - -datasets: - - from: dremio:datasets.dremio_dataset - name: dremio_dataset - params: - dremio_endpoint: grpc://1.2.3.4:32010 - dremio_username: ${env:SPICE_DREMIO_USERNAME} - dremio_password: ${env:SPICE_DREMIO_PASSWORD} -``` - -### Kubernetes secrets - -```bash -kubectl create secret generic dremio \ - --from-literal=username='demo' \ - --from-literal=password='demo1234' -``` - -`spicepod.yaml` ```yaml -version: v1beta1 -kind: Spicepod -name: spice-app - -secrets: - - from: kubernetes:dremio - name: dremio - -datasets: - - from: dremio:datasets.dremio_dataset - name: dremio_dataset - params: - dremio_endpoint: grpc://1.2.3.4:32010 - dremio_username: ${dremio:username} - dremio_password: ${dremio:password} +- from: dremio:datasets.dremio_dataset + name: dremio_dataset + params: + dremio_endpoint: grpc://127.0.0.1:32010 + dremio_username: demo + dremio_password: ${secrets:my_dremio_pass} ``` -## Using secrets - -There are currently three supported [secret stores](/components/secret-stores/index.md): +## Secrets -* [Environment variables](/components/secret-stores/env) -* [Kubernetes Secret Store](/components/secret-stores/kubernetes) -* [Keyring Secret Store](/components/secret-stores/keyring) +Spice integrates with multiple secret stores to help manage sensitive data securely. For detailed information on supported secret stores, refer to the [secret stores documentation](/components/secret-stores). Additionally, learn how to use referenced secrets in component parameters by visiting the [using referenced secrets guide](/components/secret-stores#using-secrets). From 7a7f1e4c016f302c439e3a192fd620b36342f9d8 Mon Sep 17 00:00:00 2001 From: Scott Lyons Date: Thu, 14 Nov 2024 10:07:54 -0800 Subject: [PATCH 3/4] Update spiceaidocs/docs/components/data-connectors/dremio.md Co-authored-by: Phillip LeBlanc --- spiceaidocs/docs/components/data-connectors/dremio.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spiceaidocs/docs/components/data-connectors/dremio.md b/spiceaidocs/docs/components/data-connectors/dremio.md index e92acf7b..e91d294c 100644 --- a/spiceaidocs/docs/components/data-connectors/dremio.md +++ b/spiceaidocs/docs/components/data-connectors/dremio.md @@ -6,7 +6,7 @@ description: 'Dremio Data Connector Documentation' [Dremio](https://www.dremio.com/) is a data lake engine that enables high-performance SQL queries directly on data lake storage. It provides a unified interface for querying and analyzing data from various sources without the need for complex data movement or transformation. -This connector enables using Dremio as a Data source for federated SQL queries. +This connector enables using Dremio as a data source for federated/accelerated SQL queries. ```yaml - from: dremio:datasets.dremio_dataset From 987153f13ea627fd160d55738ea2e1788095aa31 Mon Sep 17 00:00:00 2001 From: Scott Lyons Date: Thu, 14 Nov 2024 10:08:00 -0800 Subject: [PATCH 4/4] Update spiceaidocs/docs/components/data-connectors/dremio.md Co-authored-by: Phillip LeBlanc --- spiceaidocs/docs/components/data-connectors/dremio.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spiceaidocs/docs/components/data-connectors/dremio.md b/spiceaidocs/docs/components/data-connectors/dremio.md index e91d294c..5a00569d 100644 --- a/spiceaidocs/docs/components/data-connectors/dremio.md +++ b/spiceaidocs/docs/components/data-connectors/dremio.md @@ -25,7 +25,7 @@ The `from` field takes the form `dremio:dataset` where `dataset` is the fully qu :::warning [Limitations] -Only three nested levels of a dataset name is currently supported, e.g `a.b.c`. Any futher than that is not supported at this time. +Currently, only up to three levels of nesting are supported for dataset names (e.g., a.b.c). Additional levels are not supported at this time. :::