Add support for Kibana knowledge base entry assets (elastic#807)

Knowledge base entries are a new type of content that provides information for AI assistants.

A knowledge base entry in a package is composed of 3 parts:
* A manifest.yml file
* A fields folder, containing one or more fields files
* A content folder, containing one or more content files, containing the ES documents of
  the knowledge base in an NDJSON format.

code/go/internal/validator/content.go

@@ -25,6 +25,7 @@ func validateContentType(fsys fs.FS, path string, contentType spectypes.ContentT
 			}
 		}
 	case "application/json":
+	case "application/x-ndjson":
 	case "text/markdown":
 	case "text/plain":
 	default:

code/go/pkg/validator/validator_test.go

@@ -44,6 +44,7 @@ func TestValidateFile(t *testing.T) {
 		"custom_ilm_policy":                  {},
 		"profiling_symbolizer":               {},
 		"logs_synthetic_mode":                {},
+		"knowledge_base":                     {},
 		"bad_additional_content": {
 			"bad-bad",
 			[]string{
@@ -227,6 +228,13 @@ func TestValidateFile(t *testing.T) {
 				`field data_stream.vars.data_stream.dataset: Does not match pattern '^[a-zA-Z0-9]+[a-zA-Z0-9\._]*$'`,
 			},
 		},
+		"bad_knowledge_base": {
+			"index_data/foo/manifest.yml",
+			[]string{
+				`field (root): index is required`,
+				`field (root): Additional property unknown is not allowed`,
+			},
+		},
 	}
 
 	for pkgName, test := range tests {

spec/changelog.yml

@@ -7,6 +7,12 @@
     - description: Add support for content packages.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/777
+    - description: Add support for "Kibana knowledge base entry" assets.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/807
+    - description: Add support for semantic_text field definition
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/807
 - version: 3.3.0-next
   changes:
     - description: Add support for `slo` assets.

spec/content/index_data/manifest.spec.yml

@@ -0,0 +1,69 @@
+##
+## Describes the specification for a Kibana knowledge base entry's manifest.yml file
+##
+spec:
+  type: object
+  additionalProperties: false
+  properties:
+    type:
+      type: string
+      description: >
+        Type of the index data.
+      enum:
+        - knowledge_base_entry
+    description:
+      description: A description of what the index data provides
+      type: string
+
+    # Conditional properties.
+    title: true
+    index: true
+    retrieval: true
+  allOf:
+    - if:
+        properties:
+          type:
+            const: knowledge_base_entry
+        required:
+          - type
+      then:
+        properties:
+          title:
+            description: The title of the knowledge base entry as used by the knowledge base
+            type: string
+          index:
+            type: object
+            additionalProperties: false
+            properties:
+              system:
+                description: Specify whether the index should be system-managed or not
+                type: boolean
+            required:
+              - system
+          retrieval:
+            type: object
+            additionalProperties: false
+            properties:
+              syntactic_fields:
+                description: List of fields that should be used for syntactic search during retrieval.
+                type: array
+                items:
+                  type: string
+              semantic_fields:
+                description: List of fields that should be used for semantic search during retrieval.
+                type: array
+                items:
+                  type: string
+            required:
+              - syntactic_fields
+              - semantic_fields
+        required:
+          - index
+          - retrieval
+      else:
+        not:
+          required:
+            - index
+            - retrieval
+  required:
+    - type

spec/content/index_data/spec.yml

@@ -0,0 +1,33 @@
+spec:
+  additionalContents: false
+  totalContentsLimit: 50
+  contents:
+    - description: Folder containing a single index data definition
+      type: folder
+      pattern: '^[a-z0-9_]+[a-z0-9]$'
+      required: true
+      additionalContents: false
+      contents:
+        - description: An index data manifest file
+          type: file
+          contentMediaType: "application/x-yaml"
+          sizeLimit: 2MB
+          name: "manifest.yml"
+          required: true
+          $ref: "./manifest.spec.yml"
+        - description: Folder containing field definitions for the index
+          type: folder
+          name: fields
+          required: true
+          $ref: "../../integration/data_stream/fields/spec.yml"
+        - description: Folder containing content files
+          type: folder
+          name: content
+          required: true
+          additionalContents: false
+          contents:
+            - description: An index data's content file
+              type: file
+              sizeLimit: 20MB
+              contentMediaType: "application/x-ndjson"
+              pattern: '^content(-[a-z0-9]+)?\.ndjson$'

spec/content/spec.yml

@@ -46,6 +46,11 @@ spec:
     name: kibana
     required: false
     $ref: "./kibana/spec.yml"
+  - description: Folder containing Index data assets used by the package
+    type: folder
+    name: index_data
+    required: false
+    $ref: "./index_data/spec.yml"
   - description: Configuration file to process the results returned from the package validation. This file is just for package validation and it should be ignored when installing or using the package.
     type: file
     contentMediaType: "application/x-yaml"

spec/integration/data_stream/fields/fields.spec.yml

@@ -91,6 +91,7 @@ spec:
         - version
         - unsigned_long
         - counted_keyword
+        - semantic_text
 
       description:
         description: Short description of field
@@ -423,6 +424,12 @@ spec:
         type: boolean
         default: true
 
+      inference_id:
+        description: >
+          For semantic_text fields, this specifies the id of the inference
+          endpoint associated with the field
+        type: string
+
       # Conditional properties.
       default_metric: true
       metrics: true
@@ -590,12 +597,30 @@ spec:
                 - object
           required:
             - type
+      - if:
+          required:
+            - inference_id
+        then:
+          properties:
+            type:
+              enum:
+                - semantic_text
+          required:
+            - type
 
     required:
     - name
 
 # JSON patches for newer versions should be placed on top
 versions:
+  - before: 3.4.0
+    patch:
+      - op: remove
+        path: "/items/properties/type/enum/35" #remove semantic_text type
+      - op: remove
+        path: "/items/allOf/9" # removing inference_id when type is semantic_text
+      - op: remove
+        path: "/items/properties/inference_id" # removing inference_id field
   - before: 3.2.0
     patch:
       - op: remove

spec/integration/kibana/spec.yml

@@ -135,7 +135,7 @@ spec:
       contentMediaType: "application/json"
       pattern: '^{PACKAGE_NAME}-.+\.json$'
       forbiddenPatterns:
-        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden 
+        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
 versions:
   - before: 3.3.0
     patch:

test/packages/bad_knowledge_base/LICENSE.txt

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor’s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

test/packages/bad_knowledge_base/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.1.0
+  changes:
+    - description: Initial release
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/807

test/packages/bad_knowledge_base/docs/README.md

@@ -0,0 +1 @@
+This is a template for the package README.