Add vendoring to Google Cloud Functions again (elastic#21070)

## What does this PR do?

This PR adds the vendor folder back the Cloud Functions of Functionbeat, so they can be used on GCP. This PR basically reverts the removal of the vendor builder functionality.

## Why is it important?

The functions are currently broken, as there is no go.mod file and vendor folder. However, we cannot use go modules, because the Golang environment on GCP runs go version 1.13. Thus, we have to fall back to vendoring.

## Checklist

- [x] My code follows the style guidelines of this project
- [x] I have commented my code, particularly in hard-to-understand areas
~~- [ ] I have made corresponding changes to the documentation~~
~~- [ ] I have made corresponding change to the default configuration files~~
~~- [ ] I have added tests that prove my fix is effective or that my feature works~~
- [x] I have added an entry in `CHANGELOG.next.asciidoc` or `CHANGELOG-developer.next.asciidoc`.

## Related issues

Closes elastic#20830

CHANGELOG.next.asciidoc

@@ -357,6 +357,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Fix timeout option of GCP functions. {issue}16282[16282] {pull}16287[16287]
 - Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
+- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]
 
 ==== Added
 

dev-tools/mage/gotool/go.go

@@ -85,6 +85,25 @@ func ListDeps(pkg string) ([]string, error) {
 	return getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg))
 }
 
+// ListDepsLocation calls `go list -dep` for every package spec given.
+func ListDepsLocation(pkg string) (map[string]string, error) {
+	const tmpl = `{{if not .Standard}}{{.ImportPath}};{{.Dir}}{{end}}`
+
+	lines, err := getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg))
+	if err != nil {
+		return nil, err
+	}
+	deps := make(map[string]string, len(lines))
+	for _, l := range lines {
+		parts := strings.Split(l, ";")
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid number of parts")
+		}
+		deps[parts[0]] = parts[1]
+	}
+	return deps, nil
+}
+
 // ListTestFiles lists all go and cgo test files available in a package.
 func ListTestFiles(pkg string) ([]string, error) {
 	const tmpl = `{{ range .TestGoFiles }}{{ printf "%s\n" . }}{{ end }}` +

x-pack/functionbeat/dev-tools/packaging/packages.yml

@@ -65,6 +65,12 @@ shared:
       pkg/functionbeat-aws:
         source: 'provider/aws/build/golang-crossbuild/aws-linux-amd64'
         mode: 0755
+      pkg/pubsub/vendor:
+        source: 'provider/gcp/build/pubsub/vendor'
+        mode: 0644
+      pkg/storage/vendor:
+        source: 'provider/gcp/build/storage/vendor'
+        mode: 0644
       pkg/pubsub/pubsub.go:
         source: 'provider/gcp/pubsub/pubsub.go'
         mode: 0655

x-pack/functionbeat/magefile.go

@@ -159,9 +159,11 @@ func BuildPkgForFunctions() error {
 	err := os.RemoveAll("pkg")
 
 	filesToCopy := map[string]string{
-		filepath.Join("provider", "aws", "functionbeat-aws"):      filepath.Join("pkg", "functionbeat-aws"),
-		filepath.Join("provider", "gcp", "pubsub", "pubsub.go"):   filepath.Join("pkg", "pubsub", "pubsub.go"),
-		filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"),
+		filepath.Join("provider", "aws", "functionbeat-aws"):           filepath.Join("pkg", "functionbeat-aws"),
+		filepath.Join("provider", "gcp", "pubsub", "pubsub.go"):        filepath.Join("pkg", "pubsub", "pubsub.go"),
+		filepath.Join("provider", "gcp", "storage", "storage.go"):      filepath.Join("pkg", "storage", "storage.go"),
+		filepath.Join("provider", "gcp", "build", "pubsub", "vendor"):  filepath.Join("pkg", "pubsub", "vendor"),
+		filepath.Join("provider", "gcp", "build", "storage", "vendor"): filepath.Join("pkg", "storage", "vendor"),
 	}
 	for src, dest := range filesToCopy {
 		c := &devtools.CopyTask{

x-pack/functionbeat/manager/gcp/template_builder.go

@@ -190,5 +190,6 @@ func zipResources() map[string][]bundle.Resource {
 
 func zipResourcesOfFunc(typeName string) []bundle.Resource {
 	root := filepath.Join("pkg", typeName)
-	return []bundle.Resource{&bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755}}
+	vendor := bundle.Folder(filepath.Join("pkg", typeName, "vendor"), filepath.Join("pkg", typeName), 0644)
+	return append(vendor, &bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755})
 }

x-pack/functionbeat/scripts/mage/update.go

@@ -5,9 +5,13 @@
 package mage
 
 import (
+	"os"
+	"path/filepath"
+
 	"github.com/magefile/mage/mg"
 
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 // Update target namespace.
@@ -20,7 +24,7 @@ var Aliases = map[string]interface{}{
 
 // All updates all generated content.
 func (Update) All() {
-	mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs)
+	mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs, Update.VendorBeats)
 }
 
 // Config generates both the short and reference configs.
@@ -46,3 +50,39 @@ func (Update) IncludeFields() error {
 
 	return devtools.GenerateAllInOneFieldsGo()
 }
+
+// VendorBeats collects the vendor folder required to deploy the function for GCP.
+func (Update) VendorBeats() error {
+	for _, f := range []string{"pubsub", "storage"} {
+		gcpVendorPath := filepath.Join("provider", "gcp", "build", f, "vendor")
+		err := os.RemoveAll(gcpVendorPath)
+		if err != nil {
+			return err
+		}
+
+		deps, err := gotool.ListDepsLocation("github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/" + f)
+		if err != nil {
+			return err
+		}
+
+		for importPath, location := range deps {
+			cp := &devtools.CopyTask{
+				Source:  location,
+				Dest:    filepath.Join(gcpVendorPath, importPath),
+				Mode:    0600,
+				DirMode: os.ModeDir | 0750,
+				Exclude: []string{
+					".*_test.go$",
+					".*.yml",
+				},
+			}
+			err = cp.Execute()
+			if err != nil {
+				return err
+			}
+		}
+
+	}
+
+	return nil
+}