Merge remote-tracking branch 'origin/main' into check-allowed-values

.ci/Jenkinsfile

@@ -121,6 +121,8 @@ pipeline {
             dir("${BASE_DIR}"){
               withMageEnv(){
                 withCredentials([string(credentialsId: "${GITHUB_TOKEN_CREDENTIALS}", variable: 'GITHUB_TOKEN')]) {
+                  // Ensure that tags are present so goreleaser can build the changelog from the last release.
+                  gitCmd(cmd: 'fetch', args: '--unshallow --tags')
                   sh 'curl -sL https://git.io/goreleaser | bash'
                 }
               }

.ci/package-storage-publish.groovy

@@ -74,7 +74,7 @@ pipeline {
   }
   post {
     cleanup {
-      notifyBuildResult(prComment: true)
+      notifyBuildResult(prComment: false)
     }
   }
 }

README.md

@@ -86,6 +86,16 @@ Built packages can also be published to the global package registry service.
 
 For details on how to enable dependency management, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/dependency_management.md).
 
+### `elastic-package changelog`
+
+_Context: package_
+
+Use this command to work with the changelog of the package.
+
+You can use this command to modify the changelog following the expected format and good practices.
+This can be useful when introducing changelog entries for changes done by automated processes.
+
+
 ### `elastic-package check`
 
 _Context: package_

cmd/changelog.go

@@ -0,0 +1,174 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/Masterminds/semver"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/elastic-package/internal/packages/changelog"
+)
+
+const changelogLongDescription = `Use this command to work with the changelog of the package.
+
+You can use this command to modify the changelog following the expected format and good practices.
+This can be useful when introducing changelog entries for changes done by automated processes.
+`
+
+const changelogAddLongDescription = `Use this command to add an entry to the changelog file.
+
+The entry added will include the given description, type and link. It is added on top of the
+last entry in the current version
+
+Alternatively, you can start a new version indicating the specific version, or if it should
+be the next major, minor or patch version.
+`
+
+func setupChangelogCommand() *cobraext.Command {
+	addChangelogCmd := &cobra.Command{
+		Use:   "add",
+		Short: "Add an entry to the changelog file",
+		Long:  changelogAddLongDescription,
+		RunE:  changelogAddCmd,
+	}
+	addChangelogCmd.Flags().String(cobraext.ChangelogAddNextFlagName, "", cobraext.ChangelogAddNextFlagDescription)
+	addChangelogCmd.Flags().String(cobraext.ChangelogAddVersionFlagName, "", cobraext.ChangelogAddVersionFlagDescription)
+	addChangelogCmd.Flags().String(cobraext.ChangelogAddDescriptionFlagName, "", cobraext.ChangelogAddDescriptionFlagDescription)
+	addChangelogCmd.MarkFlagRequired(cobraext.ChangelogAddDescriptionFlagName)
+	addChangelogCmd.Flags().String(cobraext.ChangelogAddTypeFlagName, "", cobraext.ChangelogAddTypeFlagDescription)
+	addChangelogCmd.MarkFlagRequired(cobraext.ChangelogAddTypeFlagName)
+	addChangelogCmd.Flags().String(cobraext.ChangelogAddLinkFlagName, "", cobraext.ChangelogAddLinkFlagDescription)
+	addChangelogCmd.MarkFlagRequired(cobraext.ChangelogAddLinkFlagName)
+
+	cmd := &cobra.Command{
+		Use:   "changelog",
+		Short: "Utilities to work with the changelog of the package",
+		Long:  changelogLongDescription,
+	}
+	cmd.AddCommand(addChangelogCmd)
+
+	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
+}
+
+func changelogAddCmd(cmd *cobra.Command, args []string) error {
+	packageRoot, err := packages.MustFindPackageRoot()
+	if err != nil {
+		return errors.Wrap(err, "locating package root failed")
+	}
+
+	version, _ := cmd.Flags().GetString(cobraext.ChangelogAddVersionFlagName)
+	nextMode, _ := cmd.Flags().GetString(cobraext.ChangelogAddNextFlagName)
+	if version != "" && nextMode != "" {
+		return errors.Errorf("flags %q and %q cannot be used at the same time",
+			cobraext.ChangelogAddVersionFlagName,
+			cobraext.ChangelogAddNextFlagName)
+	}
+	if version == "" {
+		v, err := changelogCmdVersion(nextMode, packageRoot)
+		if err != nil {
+			return err
+		}
+		version = v.String()
+	}
+
+	description, _ := cmd.Flags().GetString(cobraext.ChangelogAddDescriptionFlagName)
+	changeType, _ := cmd.Flags().GetString(cobraext.ChangelogAddTypeFlagName)
+	link, _ := cmd.Flags().GetString(cobraext.ChangelogAddLinkFlagName)
+
+	entry := changelog.Revision{
+		Version: version,
+		Changes: []changelog.Entry{
+			{
+				Description: description,
+				Type:        changeType,
+				Link:        link,
+			},
+		},
+	}
+
+	err = patchChangelogFile(packageRoot, entry)
+	if err != nil {
+		return err
+	}
+
+	err = setManifestVersion(packageRoot, version)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func changelogCmdVersion(nextMode, packageRoot string) (*semver.Version, error) {
+	revisions, err := changelog.ReadChangelogFromPackageRoot(packageRoot)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read current changelog")
+	}
+	if len(revisions) == 0 {
+		return semver.MustParse("0.0.0"), nil
+	}
+
+	version, err := semver.NewVersion(revisions[0].Version)
+	if err != nil {
+		return nil, errors.Wrapf(err, "invalid version in changelog %q", revisions[0].Version)
+	}
+
+	switch nextMode {
+	case "":
+		break
+	case "major":
+		v := version.IncMajor()
+		version = &v
+	case "minor":
+		v := version.IncMinor()
+		version = &v
+	case "patch":
+		v := version.IncPatch()
+		version = &v
+	default:
+		return nil, errors.Errorf("invalid value for %q: %s",
+			cobraext.ChangelogAddNextFlagName, nextMode)
+	}
+
+	return version, nil
+}
+
+// patchChangelogFile looks for the proper place to add the new revision in the changelog,
+// trying to conserve original format and comments.
+func patchChangelogFile(packageRoot string, patch changelog.Revision) error {
+	changelogPath := filepath.Join(packageRoot, changelog.PackageChangelogFile)
+	d, err := ioutil.ReadFile(changelogPath)
+	if err != nil {
+		return err
+	}
+
+	d, err = changelog.PatchYAML(d, patch)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(changelogPath, d, 0644)
+}
+
+func setManifestVersion(packageRoot string, version string) error {
+	manifestPath := filepath.Join(packageRoot, packages.PackageManifestFile)
+	d, err := ioutil.ReadFile(manifestPath)
+	if err != nil {
+		return err
+	}
+
+	d, err = changelog.SetManifestVersion(d, version)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(manifestPath, d, 0644)
+}

cmd/root.go

@@ -16,6 +16,7 @@ import (
 
 var commands = []*cobraext.Command{
 	setupBuildCommand(),
+	setupChangelogCommand(),
 	setupCheckCommand(),
 	setupCleanCommand(),
 	setupCreateCommand(),

internal/cobraext/const.go

@@ -9,6 +9,21 @@ const (
 	BuildZipFlagName        = "zip"
 	BuildZipFlagDescription = "archive the built package"
 
+	ChangelogAddNextFlagName        = "next"
+	ChangelogAddNextFlagDescription = "changelog entry is added in the next `major`, `minor` or `patch` version"
+
+	ChangelogAddVersionFlagName        = "version"
+	ChangelogAddVersionFlagDescription = "changelog entry is added in the given version"
+
+	ChangelogAddDescriptionFlagName        = "description"
+	ChangelogAddDescriptionFlagDescription = "description for the changelog entry"
+
+	ChangelogAddTypeFlagName        = "type"
+	ChangelogAddTypeFlagDescription = "type of change (bugfix, enhancement or breaking-change) for the changelog entry"
+
+	ChangelogAddLinkFlagName        = "link"
+	ChangelogAddLinkFlagDescription = "link to the pull request or issue with more information about the changelog entry"
+
 	CheckConditionFlagName        = "check-condition"
 	CheckConditionFlagDescription = "check if the condition is met for the package, but don't install the package (e.g. kibana.version=7.10.0)"
 

internal/fields/dependency_manager.go

@@ -229,6 +229,10 @@ func transformImportedField(fd FieldDefinition) common.MapStr {
 		m["description"] = fd.Description
 	}
 
+	if fd.Pattern != "" {
+		m["pattern"] = fd.Pattern
+	}
+
 	if fd.Index != nil {
 		m["index"] = *fd.Index
 	}

internal/fields/dependency_manager_test.go

@@ -160,6 +160,25 @@ func TestDependencyManagerInjectExternalFields(t *testing.T) {
 			changed: true,
 			valid:   true,
 		},
+		{
+			title: "external with pattern",
+			defs: []common.MapStr{
+				{
+					"name":     "source.mac",
+					"external": "test",
+				},
+			},
+			result: []common.MapStr{
+				{
+					"name":        "source.mac",
+					"type":        "keyword",
+					"description": "MAC address of the source.",
+					"pattern":     "^[A-F0-9]{2}(-[A-F0-9]{2}){5,}$",
+				},
+			},
+			changed: true,
+			valid:   true,
+		},
 		{
 			title: "override not indexed external",
 			defs: []common.MapStr{
@@ -228,6 +247,12 @@ func TestDependencyManagerInjectExternalFields(t *testing.T) {
 			Index:       &indexFalse,
 			DocValues:   &indexFalse,
 		},
+		{
+			Name:        "source.mac",
+			Description: "MAC address of the source.",
+			Pattern:     "^[A-F0-9]{2}(-[A-F0-9]{2}){5,}$",
+			Type:        "keyword",
+		},
 	}}
 	dm := &DependencyManager{schema: schema}
 

internal/fields/model.go

@@ -134,16 +134,22 @@ func (fds *FieldDefinitions) UnmarshalYAML(value *yaml.Node) error {
 				return err
 			}
 
-			// "base" group is used by convention in ECS to include
-			// fields that can appear in the root level of the document.
-			// Append its child fields directly instead.
-			if name == "base" {
-				fields = append(fields, field.Fields...)
-			} else {
-				field.Name = name
-				cleanNestedNames(field.Name, field.Fields)
-				fields = append(fields, field)
+			field.Name = name
+			baseFields := cleanNested(&field)
+			if len(baseFields) > 0 {
+				// Some groups are used by convention in ECS to include
+				// fields that can appear in the root level of the document.
+				// Append their child fields directly instead.
+				// Examples of such groups are `base` or `tracing`.
+				fields = append(fields, baseFields...)
+				if len(field.Fields) == 0 {
+					// If it had base fields, and doesn't have any other
+					// field, don't add it.
+					continue
+				}
 			}
+
+			fields = append(fields, field)
 		}
 		*fds = fields
 		return nil
@@ -155,12 +161,32 @@ func (fds *FieldDefinitions) UnmarshalYAML(value *yaml.Node) error {
 	}
 }
 
-func cleanNestedNames(parent string, fields []FieldDefinition) {
-	for i := range fields {
-		if strings.HasPrefix(fields[i].Name, parent+".") {
-			fields[i].Name = fields[i].Name[len(parent)+1:]
+// cleanNested processes fields nested inside another field, and returns
+// defined base fields.
+// If a field name is prefixed by the parent field, this part is removed,
+// so the full path, taking into account the parent name, matches.
+// If a field name is not prefixed by the parent field, this is considered
+// a base field, that should appear at the top-level. It is removed from
+// the list of nested fields and returned as base field.
+func cleanNested(parent *FieldDefinition) (base []FieldDefinition) {
+	var nested []FieldDefinition
+	for _, field := range parent.Fields {
+		// If the field name is prefixed by the name of its parent,
+		// this is a normal nested field. If not, it is a base field.
+		if strings.HasPrefix(field.Name, parent.Name+".") {
+			field.Name = field.Name[len(parent.Name)+1:]
+			nested = append(nested, field)
+		} else {
+			base = append(base, field)
 		}
 	}
+
+	// At the moment of writing this code, a group field has base fields
+	// (`base` and `tracing` groups), or nested fields, but not both.
+	// This code handles the case of having groups with both kinds of fields,
+	// just in case this happens.
+	parent.Fields = nested
+	return base
 }
 
 // AllowedValues is the list of allowed values for a field.

internal/fields/validate.go

@@ -387,14 +387,20 @@ func (v *Validator) parseElementValue(key string, definition FieldDefinition, va
 				return err
 			}
 		case "date":
-			var valStr string
-			valStr, valid = val.(string)
-			if !valid {
-				break
-			}
-
-			if err := ensurePatternMatches(key, valStr, definition.Pattern); err != nil {
-				return err
+			switch val := val.(type) {
+			case string:
+				if err := ensurePatternMatches(key, val, definition.Pattern); err != nil {
+					return err
+				}
+				valid = true
+			case float64:
+				// date as seconds or milliseconds since epoch
+				if definition.Pattern != "" {
+					return fmt.Errorf("numeric date in field %q, but pattern defined", key)
+				}
+				valid = true
+			default:
+				valid = false
 			}
 		case "ip":
 			var valStr string