Documentation for new datasets[].columns[].

spiceaidocs/docs/api/http/search.md

@@ -23,12 +23,14 @@ Spicepod
 embeddings:
   - name: embedding_maker
     from: openai
+
 datasets:
   - name: app_messages
     from: file://my.csv
-    embeddings:
-      - column: document_text
-        use: embedding_maker
+    columns:
+      - name: document_text
+        embeddings:
+          - from: embedding_maker
 ```
 
 Request

spiceaidocs/docs/features/search/index.md

@@ -50,9 +50,10 @@ datasets:
     name: spiceai.issues
     acceleration:
       enabled: true
-    embeddings:
-      - column: body # The text column in the `spiceai.issues` dataset
-        use: local_embedding_model # Embedding model used for this column
+    columns:
+      - name: body
+        embeddings:
+          - from: local_embedding_model  # Embedding model used for this column
 ```
 
 By defining embeddings on the `body` column, Spice is now configured to execute similarity searches on the dataset.
@@ -85,7 +86,7 @@ datasets:
       enabled: true
     embeddings:
       - column: body
-        use: local_embedding_model
+        from: local_embedding_model
         chunking:
           enabled: true
           target_chunk_size: 512

spiceaidocs/docs/reference/spicepod/datasets.md

@@ -353,6 +353,64 @@ datasets:
         hash: upsert
 ```
 
+## `columns`
+
+Optional. Define metadata and features for specific columns in the dataset.
+
+```yaml
+datasets:
+  - from: file:sales_data.parquet
+    name: sales
+    columns:
+      - name: address_line1
+        description: The first line of the address.
+        embeddings:
+          - from: hf_minilm
+            row_id: order_number
+            chunking:
+              enabled: true
+              target_chunk_size: 256
+              overlap_size: 32
+```
+
+## `columns[*].name`
+
+The name of the column in the table schema.
+
+## `columns[*].description`
+
+Optional. A description of the column's contents and purpose.
+
+## `columns[*].embeddings`
+
+Optional. Create vector embeddings for this column.
+
+## `columns[*].embeddings[*].from`
+
+The embedding model to use, specify the component name.
+
+## `columns[*].embeddings[*].row_id`
+
+Optional. For datasets without a primary key, explicitly specify column(s) that uniquely identify a row.
+
+## `columns[*].embeddings[*].chunking`
+
+Optional. The configuration to enable and define the chunking strategy for the embedding column.
+
+```yaml
+columns:
+  - name: description
+    embeddings:
+      - from: hf_minilm
+        chunking:
+          enabled: true
+          target_chunk_size: 512
+          overlap_size: 128
+          trim_whitespace: false
+```
+
+See [`embeddings[*].chunking`](#embeddingschunking) for details.
+
 ## `embeddings`
 
 Optional. Create vector embeddings for specific columns of the dataset.

spicepod.yaml

@@ -18,22 +18,22 @@ datasets:
       github_client_id: ${secrets:GITHUB_CLIENT_ID}
       github_private_key: ${secrets:GITHUB_PRIVATE_KEY}
       github_installation_id: ${secrets:GITHUB_INSTALLATION_ID}
-      include: 'spiceaidocs/docs/**/*.md'
+      include: "spiceaidocs/docs/**/*.md"
     acceleration:
       enabled: true
       refresh_check_interval: 4h
       refresh_jitter_enabled: true
       refresh_jitter_max: 30m
-    embeddings:
-      - column: content
-        column_pk:
-          - path
-        use: openai_embeddings
-        chunking:
-          enabled: true
-          target_chunk_size: 512
-          overlap_size: 128
-          trim_whitespace: true
+    columns:
+      - name: content
+        embeddings:
+          from: openai_embeddings
+          row_id: path
+          chunking:
+            enabled: true
+            target_chunk_size: 512
+            overlap_size: 128
+            trim_whitespace: true
 
   - from: github:github.com/spiceai/samples/files/trunk
     name: spiceai.samples
@@ -45,22 +45,22 @@ datasets:
       github_client_id: ${secrets:GITHUB_CLIENT_ID}
       github_private_key: ${secrets:GITHUB_PRIVATE_KEY}
       github_installation_id: ${secrets:GITHUB_INSTALLATION_ID}
-      include: '**/*.md'
+      include: "**/*.md"
     acceleration:
       enabled: true
       refresh_check_interval: 4h
       refresh_jitter_enabled: true
       refresh_jitter_max: 30m
-    embeddings:
-      - column: content
-        column_pk:
-          - path
-        use: openai_embeddings
-        chunking:
-          enabled: true
-          target_chunk_size: 512
-          overlap_size: 128
-          trim_whitespace: true
+    columns:
+      - name: content
+        embeddings:
+          from: openai_embeddings
+          row_id: path
+          chunking:
+            enabled: true
+            target_chunk_size: 512
+            overlap_size: 128
+            trim_whitespace: true
 
   - from: github:github.com/spiceai/quickstarts/files/trunk
     name: spiceai.quickstarts
@@ -73,22 +73,22 @@ datasets:
       github_client_id: ${secrets:GITHUB_CLIENT_ID}
       github_private_key: ${secrets:GITHUB_PRIVATE_KEY}
       github_installation_id: ${secrets:GITHUB_INSTALLATION_ID}
-      include: '**/*.md'
+      include: "**/*.md"
     acceleration:
       enabled: true
       refresh_check_interval: 4h
       refresh_jitter_enabled: true
       refresh_jitter_max: 30m
-    embeddings:
-      - column: content
-        column_pk:
-          - path
-        use: openai_embeddings
-        chunking:
-          enabled: true
-          target_chunk_size: 512
-          overlap_size: 128
-          trim_whitespace: true
+    columns:
+      - name: content
+        embeddings:
+          from: openai_embeddings
+          row_id: path
+          chunking:
+            enabled: true
+            target_chunk_size: 512
+            overlap_size: 128
+            trim_whitespace: true
 
   - from: github:github.com/spiceai/blog/files/trunk
     name: spiceai.blog
@@ -102,22 +102,22 @@ datasets:
       github_client_id: ${secrets:GITHUB_CLIENT_ID}
       github_private_key: ${secrets:GITHUB_PRIVATE_KEY}
       github_installation_id: ${secrets:GITHUB_INSTALLATION_ID}
-      include: 'content/posts/**/*.md'
+      include: "content/posts/**/*.md"
     acceleration:
       enabled: true
       refresh_check_interval: 4h
       refresh_jitter_enabled: true
       refresh_jitter_max: 30m
-    embeddings:
-      - column: content
-        column_pk:
-          - path
-        use: openai_embeddings
-        chunking:
-          enabled: true
-          target_chunk_size: 512
-          overlap_size: 128
-          trim_whitespace: true
+    columns:
+      - name: content
+        embeddings:
+          from: openai_embeddings
+          row_id: path
+          chunking:
+            enabled: true
+            target_chunk_size: 512
+            overlap_size: 128
+            trim_whitespace: true
 
   - from: github:github.com/spiceai/spiceai/issues
     name: spiceai.issues