Write input package spec (#328)

* WIP

* Fields

* Basic spec

* Side files covered

* custom_logs covered

* All input packages are covered

* Fix: changelog

* Address PR comments

* Harden limits

* Refer to integration's changelog

* Add Version field to Package

* Fix tests

* Fix: error message

* Fix changelog

* Update code/go/internal/validator/folder_spec.go

Co-authored-by: Josh Dover <[email protected]>

* Log warning

* Spec testing

* Fix: beta and ga as valid levels

* Use switch

* Use beta

* Use technical previews

* Adjust conditions

* Add test case

* Fix

Co-authored-by: Josh Dover <[email protected]>

code/go/internal/validator/folder_spec.go

@@ -90,10 +90,10 @@ func (s *folderSpec) validate(pkg *Package, path string) ve.ValidationErrors {
 	switch s.Release {
 		case "", "ga": // do nothing
 		case "beta":
-			if pkg.Version.Prerelease() == "" {
+			if pkg.Version.Major() > 0 && pkg.Version.Prerelease() == "" {
 				errs = append(errs, errors.Errorf("spec for [%s] defines beta features which can't be enabled for packages with a stable semantic version", pkg.Path(path)))
 			} else {
-				log.Printf("Warning: package with non-stable semantic version and active beta features (enabled in [%s]) can't be released as stable version.q", pkg.Path(path))
+				log.Printf("Warning: package with non-stable semantic version and active beta features (enabled in [%s]) can't be released as stable version.", pkg.Path(path))
 			}
 		default:
 			errs = append(errs, errors.Errorf("unsupport release level, supported values: beta, ga"))

code/go/pkg/validator/validator_test.go

@@ -27,6 +27,9 @@ func TestValidateFile(t *testing.T) {
 		"deploy_terraform":    {},
 		"time_series":         {},
 		"missing_data_stream": {},
+		"custom_logs":         {},
+		"httpjson_input":      {},
+		"sql_input":           {},
 		"bad_additional_content": {
 			"bad-bad",
 			[]string{

spec.go

@@ -9,7 +9,7 @@ import (
 	"io/fs"
 )
 
-//go:embed versions/1 versions/1/integration/_dev versions/1/integration/data_stream/_dev
+//go:embed versions/1 versions/1/integration/_dev versions/1/integration/data_stream/_dev versions/1/input
 var content embed.FS
 
 // FS returns an io/fs.FS for accessing the "package-spec/version" contents.

test/packages/custom_logs/changelog.yml

@@ -1,5 +1,5 @@
 # newer versions go on top
-- version: "0.0.1"
+- version: "1.2.0-beta"
   changes:
     - description: Initial draft of the package
       type: enhancement

test/packages/custom_logs/manifest.yml

@@ -4,8 +4,7 @@ title: Custom Logs
 description: >-
   Read lines from active log files with Elastic Agent.
 type: input
-version: 1.2.3
-release: ga
+version: 1.2.0-beta
 license: basic
 categories:
   - custom

test/packages/httpjson_input/changelog.yml

@@ -1,5 +1,5 @@
 # newer versions go on top
-- version: "0.0.1"
+- version: "0.2.0"
   changes:
     - description: Initial draft of the package
       type: enhancement

test/packages/httpjson_input/manifest.yml

@@ -4,8 +4,7 @@ title: HTTPJSON Input
 description: >-
   Ingest data from custom RESTful API's that do not currently have an existing integration.
 type: input
-version: 1.2.0
-release: ga
+version: 0.2.0
 categories:
   - custom
 conditions:

test/packages/sql_input/changelog.yml

@@ -1,5 +1,5 @@
 # newer versions go on top
-- version: "0.0.1"
+- version: "0.2.0"
   changes:
     - description: Initial draft of the package
       type: enhancement

test/packages/sql_input/manifest.yml

@@ -4,8 +4,7 @@ title: SQL Input
 description: >-
   Execute custom queries against an SQL database and store the results in Elasticsearch.
 type: input
-version: 1.0.1
-release: ga
+version: 0.2.0
 license: basic
 categories:
   - custom

versions/1/changelog.yml

@@ -10,6 +10,9 @@
     - description: Define beta features in spec
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/341
+    - description: Write "input" package spec (beta)
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/328
 - version: 1.9.0
   changes:
   - description: Prepare for next version

versions/1/input/manifest.spec.yml

@@ -0,0 +1,131 @@
+##
+## Describes the specification for the input package's main manifest.yml file
+##
+spec:
+  # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
+  type: object
+  additionalProperties: false
+  properties:
+    format_version:
+      description: The version of the package specification format used by this package.
+      $ref: "../integration/manifest.spec.yml#/definitions/version"
+    name:
+      description: The name of the package.
+      type: string
+      pattern: '^[a-z0-9_]+$'
+      examples:
+      - apache
+    title:
+      description: The title of the package.
+      type: string
+      examples:
+      - Apache
+    description:
+      description: A longer description of the package.
+      type: string
+      examples:
+      - Apache Integration
+    version:
+      description: The version of the package.
+      $ref: "../integration/manifest.spec.yml#/definitions/version"
+    license:
+      description: The license under which the package is being released.
+      type: string
+      enum:
+      - basic
+      default: basic
+      examples:
+      - basic
+    type:
+      description: The type of package.
+      type: string
+      enum:
+      - input
+      examples:
+      - input
+    categories:
+      $ref: "../integration/manifest.spec.yml#/definitions/categories"
+    conditions:
+      description: Conditions under which this package can be installed.
+      type: object
+      additionalProperties: false
+      properties:
+        kibana:
+          description: Kibana conditions
+          type: object
+          additionalProperties: false
+          properties:
+            version:
+              type: string
+              description: Kibana versions compatible with this package.
+              examples:
+                - ">=7.9.0"
+    policy_templates:
+      description: List of policy templates offered by this package.
+      type: array
+      items:
+        type: object
+        additionalProperties: false
+        properties:
+          name:
+            description: Name of policy template.
+            type: string
+            examples:
+              - apache
+          type:
+            description: Type of data stream
+            type: string
+            enum:
+              - metrics
+              - logs
+              - synthetics
+              - traces
+            examples:
+              - metrics
+          title:
+            description: Title of policy template.
+            type: string
+            examples:
+              - Apache logs and metrics
+          description:
+            description: Longer description of policy template.
+            type: string
+            examples:
+              - Collect logs and metrics from Apache instances
+          icons:
+            $ref: "../integration/manifest.spec.yml#/definitions/icons"
+          screenshots:
+            $ref: "../integration/manifest.spec.yml#/definitions/screenshots"
+          vars:
+            $ref: "../integration/data_stream/manifest.spec.yml#/definitions/vars"
+          input:
+            type: string
+            examples:
+              - aws/metrics
+              - s3
+              - file
+          template_path:
+            description: "Path to Elasticsearch index template for stream."
+            type: string
+        required:
+          - name
+          - title
+          - description
+          - type
+          - input
+    icons:
+      $ref: "../integration/manifest.spec.yml#/definitions/icons"
+    screenshots:
+      $ref: "../integration/manifest.spec.yml#/definitions/screenshots"
+    vars:
+      $ref: "../integration/data_stream/manifest.spec.yml#/definitions/vars"
+    owner:
+      $ref: "../integration/manifest.spec.yml#/definitions/owner"
+  required:
+  - format_version
+  - name
+  - title
+  - description
+  - version
+  - type
+  - owner

versions/1/input/spec.yml

@@ -0,0 +1,43 @@
+##
+## Entrypoint of "input packages" specification.
+## The specification is considered "beta" at the moment, so it may change until we release it as GA.
+##
+## Describes the folders and files that make up a package.
+##
+spec:
+  additionalContents: true
+  totalContentsLimit: 65535
+  totalSizeLimit: 50MB
+  sizeLimit: 30MB
+  configurationSizeLimit: 5MB
+  relativePathSizeLimit: 3MB
+  release: beta
+  contents:
+  - description: The main package manifest file
+    type: file
+    contentMediaType: "application/x-yaml"
+    sizeLimit: 5MB
+    name: "manifest.yml"
+    required: true
+    $ref: "./manifest.spec.yml"
+  - description: The package's CHANGELOG file
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "changelog.yml"
+    required: true
+    $ref: "../integration/changelog.spec.yml"
+  - description: Folder containing agent-related definitions
+    type: folder
+    name: agent
+    required: true
+    $ref: "../integration/agent/spec.yml"
+  - description: Folder containing documentation for the package
+    type: folder
+    name: docs
+    required: true
+    $ref: "../integration/docs/spec.yml"
+  - description: Folder containing field definitions
+    type: folder
+    name: fields
+    required: true
+    $ref: "../integration/data_stream/fields/spec.yml"

versions/1/integration/data_stream/agent/spec.yml

@@ -0,0 +1,14 @@
+spec:
+  additionalContents: false
+  contents:
+  - description: Folder containing input definitions
+    type: folder
+    name: stream
+    required: true
+    additionalContents: false
+    contents:
+    - description: Folder containing agent stream definitions
+      type: file
+      sizeLimit: 2MB
+      pattern: '^.+.yml.hbs$'
+      required: true

versions/1/integration/data_stream/spec.yml

@@ -25,12 +25,7 @@ spec:
       name: agent
       required: false
       additionalContents: false
-      contents:
-      - description: Folder containing agent stream definitions
-        type: folder
-        name: stream
-        required: true
-        additionalContents: false
+      $ref: "./agent/spec.yml"
     - description: Folder containing Elasticsearch assets
       type: folder
       name: elasticsearch

versions/1/integration/manifest.spec.yml

@@ -1,5 +1,5 @@
 ##
-## Describes the specification for the package's main manifest.yml file
+## Describes the specification for the integration package's main manifest.yml file
 ##
 spec:
   # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
@@ -107,6 +107,16 @@ spec:
         - "1.0.0-beta1"
         - "1.0.0-SNAPSHOT"
         - "1.0.0-next"
+    owner:
+      type: object
+      additionalProperties: false
+      properties:
+        github:
+          description: GitHub repository name of package maintainer.
+          type: string
+          pattern: '^(([a-zA-Z0-9-]+)|([a-zA-Z0-9-]+\/[a-zA-Z0-9-]+))$'
+      required:
+        - github
   properties:
     format_version:
       description: The version of the package specification format used by this package.
@@ -268,15 +278,7 @@ spec:
     vars:
       $ref: "./data_stream/manifest.spec.yml#/definitions/vars"
     owner:
-      type: object
-      additionalProperties: false
-      properties:
-        github:
-          description: GitHub repository name of package maintainer.
-          type: string
-          pattern: '^(([a-zA-Z0-9-]+)|([a-zA-Z0-9-]+\/[a-zA-Z0-9-]+))$'
-      required:
-      - github
+      $ref: "#/definitions/owner"
     elasticsearch:
       description: Elasticsearch requirements
       type: object

versions/1/integration/spec.yml

@@ -1,5 +1,5 @@
 ##
-## Entrypoint of package specification.
+## Entrypoint of "integration packages" specification.
 ##
 ## Describes the folders and files that make up a package.
 ##