Support input packages (#809)

* Add sql_input to testdata

* Regenerate testdata

* Generate package input

* Skip inputs

* Add model changes for input packages

* Fix: changelog

* Search input types

* Update OpenAPI

* Add sql_input 1.0.2

* Test package: migrated from integration to input

* Generated

* More tests

* Move package

* Fix

* Cleanup

* Fix: prerelease

* Fix

* Fix

* Fix

CHANGELOG.md

@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 * Prepare stub for Storage Indexer. Disable fetching packages from Package Storage v1. [#811](https://github.com/elastic/package-registry/pull/811)
+* Support input packages. [#809](https://github.com/elastic/package-registry/pull/809)
 
 ### Deprecated
 

config.yml

@@ -1,4 +1,3 @@
 package_paths:
   - ./testdata/package
   - ./testdata/local-storage
-  - ./build/package-storage/packages

indexer.go

@@ -18,7 +18,7 @@ type Indexer interface {
 type CombinedIndexer []Indexer
 
 func NewCombinedIndexer(indexers ...Indexer) CombinedIndexer {
-	return CombinedIndexer(indexers)
+	return indexers
 }
 
 func (c CombinedIndexer) Init(ctx context.Context) error {

magefile.go

@@ -27,9 +27,7 @@ var (
 	// GoLicenserImportPath controls the import path used to install go-licenser.
 	GoLicenserImportPath = "github.com/elastic/go-licenser"
 
-	buildDir       = "./build"
-	storageRepoDir = filepath.Join(buildDir, "package-storage")
-	packagePaths   = []string{filepath.Join(storageRepoDir, "packages"), "./testdata/package/"}
+	buildDir = "./build"
 )
 
 func Build() error {
@@ -39,9 +37,6 @@ func Build() error {
 func Check() error {
 	Format()
 
-	// Setup the variables for the tests and not create tarGz files
-	packagePaths = []string{"testdata/package"}
-
 	err := Build()
 	if err != nil {
 		return err

main_test.go

@@ -86,6 +86,9 @@ func TestEndpoints(t *testing.T) {
 		{"/search?prerelease=true", "/search", "search-package-prerelease.json", searchHandler(indexer, testCacheTime)},
 		{"/search?prerelease=foo", "/search", "search-package-prerelease-error.txt", searchHandler(indexer, testCacheTime)},
 		{"/search?category=datastore&prerelease=true", "/search", "search-category-datastore-prerelease.json", searchHandler(indexer, testCacheTime)},
+		{"/search?type=input&prerelease=true", "/search", "search-input-packages.json", searchHandler(indexer, testCacheTime)},
+		{"/search?type=input&package=integration_input&prerelease=true", "/search", "search-input-integration-package.json", searchHandler(indexer, testCacheTime)},
+		{"/search?type=integration&package=integration_input&prerelease=true", "/search", "search-integration-integration-package.json", searchHandler(indexer, testCacheTime)},
 		{"/favicon.ico", "", "favicon.ico", faviconHandleFunc},
 
 		// Removed flags, kept to ensure that they don't break requests from old versions.
@@ -325,6 +328,7 @@ func TestPackageIndex(t *testing.T) {
 		{"/package/missing/1.0.0/", packageIndexRouterPath, "index-package-not-found.txt", packageIndexHandler},
 		{"/package/example/999.0.0/", packageIndexRouterPath, "index-package-revision-not-found.txt", packageIndexHandler},
 		{"/package/example/a.b.c/", packageIndexRouterPath, "index-package-invalid-version.txt", packageIndexHandler},
+		{"/package/sql_input/1.0.1/", packageIndexRouterPath, "sql-input-package.json", packageIndexHandler},
 	}
 
 	for _, test := range tests {

packages/package.go

@@ -106,12 +106,17 @@ type PolicyTemplate struct {
 	Title       string   `config:"title" json:"title" validate:"required"`
 	Description string   `config:"description" json:"description" validate:"required"`
 	DataStreams []string `config:"data_streams,omitempty" json:"data_streams,omitempty" yaml:"data_streams,omitempty"`
-	Inputs      []Input  `config:"inputs" json:"inputs"`
+	Inputs      []Input  `config:"inputs" json:"inputs,omitempty" yaml:"inputs,omitempty"`
 	Multiple    *bool    `config:"multiple" json:"multiple,omitempty" yaml:"multiple,omitempty"`
 	Icons       []Image  `config:"icons,omitempty" json:"icons,omitempty" yaml:"icons,omitempty"`
 	Categories  []string `config:"categories,omitempty" json:"categories,omitempty" yaml:"categories,omitempty"`
 	Screenshots []Image  `config:"screenshots,omitempty" json:"screenshots,omitempty" yaml:"screenshots,omitempty"`
 	Readme      *string  `config:"readme,omitempty" json:"readme,omitempty" yaml:"readme,omitempty"`
+
+	// For purposes of "input packages"
+	Type         string `config:"type,omitempty" json:"type,omitempty" yaml:"type,omitempty"`
+	Input        string `config:"input,omitempty" json:"input,omitempty" yaml:"input,omitempty"`
+	TemplatePath string `config:"template_path,omitempty" json:"template_path,omitempty" yaml:"template_path,omitempty"`
 }
 
 type Conditions struct {

packages/packages.go

@@ -256,6 +256,7 @@ type Filter struct {
 	KibanaVersion  *semver.Version
 	PackageName    string
 	PackageVersion string
+	PackageType    string
 
 	// Deprecated, release tags to be removed.
 	Experimental bool
@@ -297,6 +298,10 @@ func (f *Filter) Apply(ctx context.Context, packages Packages) Packages {
 			continue
 		}
 
+		if f.PackageType != "" && f.PackageType != p.Type {
+			continue
+		}
+
 		addPackage := true
 		if !f.AllVersions {
 			// Check if the version exists and if it should be added or not.

search.go

@@ -73,6 +73,10 @@ func newSearchFilterFromQuery(query url.Values) (*packages.Filter, error) {
 		filter.PackageName = v
 	}
 
+	if v := query.Get("type"); v != "" {
+		filter.PackageType = v
+	}
+
 	if v := query.Get("all"); v != "" {
 		// Default is false, also on error
 		filter.AllVersions, err = strconv.ParseBool(v)

testdata/generated/categories-experimental.json

@@ -27,12 +27,12 @@
   {
     "id": "custom",
     "title": "Custom",
-    "count": 14
+    "count": 16
   },
   {
     "id": "datastore",
     "title": "Datastore",
-    "count": 1
+    "count": 3
   },
   {
     "id": "message_queue",

testdata/generated/categories-include-policy-templates.json

@@ -12,12 +12,12 @@
   {
     "id": "custom",
     "title": "Custom",
-    "count": 12
+    "count": 13
   },
   {
     "id": "datastore",
     "title": "Datastore",
-    "count": 2
+    "count": 3
   },
   {
     "id": "web",

testdata/generated/categories-kibana652.json

@@ -12,7 +12,12 @@
   {
     "id": "custom",
     "title": "Custom",
-    "count": 6
+    "count": 7
+  },
+  {
+    "id": "datastore",
+    "title": "Datastore",
+    "count": 1
   },
   {
     "id": "message_queue",

testdata/generated/categories-prerelease-kibana652.json

@@ -12,7 +12,12 @@
   {
     "id": "custom",
     "title": "Custom",
-    "count": 6
+    "count": 7
+  },
+  {
+    "id": "datastore",
+    "title": "Datastore",
+    "count": 1
   },
   {
     "id": "message_queue",

testdata/generated/categories-prerelease.json

@@ -27,12 +27,12 @@
   {
     "id": "custom",
     "title": "Custom",
-    "count": 14
+    "count": 16
   },
   {
     "id": "datastore",
     "title": "Datastore",
-    "count": 1
+    "count": 3
   },
   {
     "id": "message_queue",

testdata/generated/categories.json

@@ -12,7 +12,12 @@
   {
     "id": "custom",
     "title": "Custom",
-    "count": 12
+    "count": 13
+  },
+  {
+    "id": "datastore",
+    "title": "Datastore",
+    "count": 1
   },
   {
     "id": "web",

testdata/generated/package/integration_input/1.0.0/index.json

@@ -0,0 +1,99 @@
+{
+  "name": "integration_input",
+  "title": "Integration input",
+  "version": "1.0.0",
+  "release": "ga",
+  "description": "This is the example integration",
+  "type": "integration",
+  "download": "/epr/integration_input/integration_input-1.0.0.zip",
+  "path": "/package/integration_input/1.0.0",
+  "conditions": {
+    "kibana": {
+      "version": "^8.4.0"
+    }
+  },
+  "owner": {
+    "github": "ruflin"
+  },
+  "categories": [
+    "crm",
+    "azure"
+  ],
+  "format_version": "1.0.0",
+  "readme": "/package/integration_input/1.0.0/docs/README.md",
+  "license": "basic",
+  "screenshots": [
+    {
+      "src": "/img/kibana-envoyproxy.jpg",
+      "path": "/package/integration_input/1.0.0/img/kibana-envoyproxy.jpg",
+      "title": "IP Tables Ubiquity Dashboard",
+      "size": "1492x1464",
+      "type": "image/png"
+    }
+  ],
+  "assets": [
+    "/package/integration_input/1.0.0/manifest.yml",
+    "/package/integration_input/1.0.0/docs/README.md",
+    "/package/integration_input/1.0.0/img/icon.png",
+    "/package/integration_input/1.0.0/img/kibana-envoyproxy.jpg",
+    "/package/integration_input/1.0.0/data_stream/foo/manifest.yml",
+    "/package/integration_input/1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
+    "/package/integration_input/1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
+    "/package/integration_input/1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
+    "/package/integration_input/1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
+    "/package/integration_input/1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
+    "/package/integration_input/1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
+    "/package/integration_input/1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
+    "/package/integration_input/1.0.0/data_stream/foo/fields/base-fields.yml",
+    "/package/integration_input/1.0.0/data_stream/foo/agent/stream/stream.yml.hbs",
+    "/package/integration_input/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-entry.json",
+    "/package/integration_input/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-http.json",
+    "/package/integration_input/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-json.json",
+    "/package/integration_input/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-plaintext.json",
+    "/package/integration_input/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-tcp.json"
+  ],
+  "policy_templates": [
+    {
+      "name": "logs",
+      "title": "Logs datasource",
+      "description": "Datasource for your log files.",
+      "inputs": [
+        {
+          "type": "foo"
+        }
+      ],
+      "multiple": true,
+      "categories": [
+        "datastore"
+      ]
+    }
+  ],
+  "data_streams": [
+    {
+      "type": "logs",
+      "dataset": "integration_input.foo",
+      "title": "Foo",
+      "release": "ga",
+      "ingest_pipeline": "pipeline-entry",
+      "streams": [
+        {
+          "input": "foo",
+          "vars": [
+            {
+              "name": "paths",
+              "type": "text",
+              "description": "Path to log files to be collected",
+              "multi": true,
+              "required": true,
+              "show_user": false
+            }
+          ],
+          "template_path": "stream.yml.hbs",
+          "enabled": true
+        }
+      ],
+      "package": "integration_input",
+      "path": "foo"
+    }
+  ]
+}

testdata/generated/package/integration_input/1.0.2/index.json

@@ -0,0 +1,61 @@
+{
+  "name": "integration_input",
+  "title": "Integration Input",
+  "version": "1.0.2",
+  "release": "ga",
+  "description": "Sample package that was an integration and got migrated to input",
+  "type": "input",
+  "download": "/epr/integration_input/integration_input-1.0.2.zip",
+  "path": "/package/integration_input/1.0.2",
+  "icons": [
+    {
+      "src": "/img/sample-logo.svg",
+      "path": "/package/integration_input/1.0.2/img/sample-logo.svg",
+      "type": "image/svg+xml"
+    }
+  ],
+  "conditions": {
+    "kibana": {
+      "version": "^8.4.0"
+    }
+  },
+  "owner": {
+    "github": "elastic/integrations"
+  },
+  "categories": [
+    "custom",
+    "datastore"
+  ],
+  "format_version": "1.0.0",
+  "readme": "/package/integration_input/1.0.2/docs/README.md",
+  "license": "basic",
+  "screenshots": [
+    {
+      "src": "/img/sample-screenshot.png",
+      "path": "/package/integration_input/1.0.2/img/sample-screenshot.png",
+      "title": "Sample screenshot",
+      "size": "600x600",
+      "type": "image/png"
+    }
+  ],
+  "assets": [
+    "/package/integration_input/1.0.2/changelog.yml",
+    "/package/integration_input/1.0.2/manifest.yml",
+    "/package/integration_input/1.0.2/docs/README.md",
+    "/package/integration_input/1.0.2/fields/input.yml",
+    "/package/integration_input/1.0.2/img/sample-logo.svg",
+    "/package/integration_input/1.0.2/img/sample-screenshot.png",
+    "/package/integration_input/1.0.2/agent/input/input.yml.hbs"
+  ],
+  "policy_templates": [
+    {
+      "name": "sql_query",
+      "title": "SQL Query",
+      "description": "Query the database to capture metrics.",
+      "multiple": true,
+      "type": "metrics",
+      "input": "sql",
+      "template_path": "input.yml.hbs"
+    }
+  ]
+}