From db8dfa34a65a736289d271a159c23c70ccddad8e Mon Sep 17 00:00:00 2001 From: Prakhar Agarwal Date: Tue, 31 Oct 2023 14:09:30 +0530 Subject: [PATCH] Add cmd collect (#6) * golangtransformer complete with transform code * fix: removed kubernetes dep * feat: add command collect --- cmd/collect.go | 88 ++++ cmd/docs.go | 109 +++++ collector/cfappscollector.go | 350 +++++++++++++ collector/cfservicescollector.go | 74 +++ collector/cfutils.go | 152 ++++++ collector/clustercollector.go | 649 +++++++++++++++++++++++++ collector/collector.go | 29 ++ collector/imagescollector.go | 178 +++++++ collector/sourcetypes/dockercompose.go | 28 ++ collector/sourcetypes/dockerinspect.go | 31 ++ common/utils.go | 91 ++++ go.mod | 10 +- go.sum | 58 +++ lib/collector.go | 63 +++ 14 files changed, 1909 insertions(+), 1 deletion(-) create mode 100644 cmd/collect.go create mode 100644 cmd/docs.go create mode 100644 collector/cfappscollector.go create mode 100644 collector/cfservicescollector.go create mode 100644 collector/cfutils.go create mode 100644 collector/clustercollector.go create mode 100644 collector/collector.go create mode 100644 collector/imagescollector.go create mode 100644 collector/sourcetypes/dockercompose.go create mode 100644 collector/sourcetypes/dockerinspect.go create mode 100644 lib/collector.go diff --git a/cmd/collect.go b/cmd/collect.go new file mode 100644 index 0000000..ff6d978 --- /dev/null +++ b/cmd/collect.go @@ -0,0 +1,88 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "os" + "path/filepath" + "strings" + + "github.com/konveyor/move2kube-wasm/lib" + "github.com/konveyor/move2kube-wasm/types" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type collectFlags struct { + annotations string + outpath string + srcpath string +} + +func collectHandler(flags collectFlags) { + var err error + annotations := flags.annotations + outpath := flags.outpath + srcpath := flags.srcpath + + if outpath != "" { + if outpath, err = filepath.Abs(outpath); err != nil { + logrus.Fatalf("Failed to make the output directory path '%s' absolute. Error: %q", outpath, err) + } + } + if srcpath != "" { + srcpath, err = filepath.Abs(srcpath) + if err != nil { + logrus.Fatalf("Failed to make the source directory path '%s' absolute. Error: %q", srcpath, err) + } + fi, err := os.Stat(srcpath) + if os.IsNotExist(err) { + logrus.Fatalf("Source directory '%s' does not exist. Error: %q", srcpath, err) + } else if err != nil { + logrus.Fatalf("Error while accessing directory: '%s' . Error: %q", srcpath, err) + } else if !fi.IsDir() { + logrus.Fatalf("Source path is a file, expected '%s' to be a directory.", srcpath) + } + } + outpath = filepath.Join(filepath.Clean(outpath), types.AppNameShort+"_collect") + if annotations == "" { + lib.Collect(srcpath, outpath, []string{}) + } else { + lib.Collect(srcpath, outpath, strings.Split(annotations, ",")) + } + logrus.Infof("Collect Output in [%s]. Copy this directory into the source directory to be used for planning.", outpath) +} + +// GetCollectCommand returns a command to collect information from running applications +func GetCollectCommand() *cobra.Command { + viper.AutomaticEnv() + + flags := collectFlags{} + collectCmd := &cobra.Command{ + Use: "collect", + Short: "Collect and process metadata from multiple sources.", + Long: "Collect metadata from multiple sources (cluster, image repo etc.), filter and summarize it into a yaml.", + Run: func(*cobra.Command, []string) { collectHandler(flags) }, + } + + collectCmd.Flags().StringVarP(&flags.annotations, "annotations", "a", "", "Specify annotations to select collector subset.") + collectCmd.Flags().StringVarP(&flags.outpath, outputFlag, "o", ".", "Specify output directory for collect.") + collectCmd.Flags().StringVarP(&flags.srcpath, sourceFlag, "s", "", "Specify source directory for the artifacts to be considered while collecting.") + + return collectCmd +} diff --git a/cmd/docs.go b/cmd/docs.go new file mode 100644 index 0000000..1566740 --- /dev/null +++ b/cmd/docs.go @@ -0,0 +1,109 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/konveyor/move2kube-wasm/common" + "github.com/konveyor/move2kube-wasm/types" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/cobra/doc" + "github.com/spf13/viper" +) + +type generateFlags struct { + outputDir string +} + +const docsTemplateStr = `--- +layout: default +title: {{ .Title }} +permalink: /commands{{ .URL }} +{{- if .Parent }} +parent: {{ .Parent -}} +{{ end }} +{{ .Extra -}} +--- +{{ .Notes -}} + +` + +func generateHandler(flags generateFlags) { + docsTemplate := template.Must(template.New("gen-docs").Parse(docsTemplateStr)) + filePrepender := func(filename string) string { + data := struct { + Title string + URL string + Parent string + Extra string + Notes string + }{} + commandsParent := "Commands" + data.Title = strings.TrimPrefix(strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename)), types.AppName+"_") + data.URL = "/" + data.Title + data.Parent = commandsParent + if data.Title == types.AppName { + data.Title = commandsParent + data.URL = "" + data.Parent = "" + data.Extra = "has_children: true\nnav_order: 4\nhas_toc: false\n" + data.Notes = "\n###### This documentation was generated by running `" + types.AppName + " docs`\n\n" + } + output := bytes.Buffer{} + if err := docsTemplate.Execute(&output, data); err != nil { + logrus.Errorf("failed to generate the documentation for the filename %s using the template. Error: %q", filename, err) + return "" + } + return output.String() + } + linkHandler := func(filename string) string { + link := strings.TrimPrefix(strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename)), types.AppName+"_") + if link == types.AppName { + return "/commands" + } + return "/commands/" + link + } + logrus.Infof("generating the documentation for all the commands into the directory at the path: %s", flags.outputDir) + if err := os.MkdirAll(flags.outputDir, common.DefaultDirectoryPermission); err != nil { + logrus.Fatalf("error while making the output directory at path %s to store the documentation. Error: %q", flags.outputDir, err) + } + rootCmd := GetRootCmd() + if err := doc.GenMarkdownTreeCustom(rootCmd, flags.outputDir, filePrepender, linkHandler); err != nil { + logrus.Fatalf("error while generating documentation. Error: %q", err) + } +} + +// GetGenerateDocsCommand returns a command to generate the documentation for all the commands +func GetGenerateDocsCommand() *cobra.Command { + viper.AutomaticEnv() + flags := generateFlags{} + generateDocsCmd := &cobra.Command{ + Hidden: true, + Use: "docs", + Short: "Generate the documentation for the commands", + Long: "Generate the documentation for the commands. The documentation is in markdown format.", + Run: func(_ *cobra.Command, __ []string) { generateHandler(flags) }, + } + generateDocsCmd.Flags().StringVarP(&flags.outputDir, "output", "o", "commands", "Path to the directory where the documentation will be generated.") + return generateDocsCmd +} diff --git a/collector/cfappscollector.go b/collector/cfappscollector.go new file mode 100644 index 0000000..ed0955e --- /dev/null +++ b/collector/cfappscollector.go @@ -0,0 +1,350 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" + + cfclient "github.com/cloudfoundry-community/go-cfclient/v2" + "github.com/konveyor/move2kube-wasm/common" + "github.com/konveyor/move2kube-wasm/types" + "github.com/pkg/errors" + + "github.com/sirupsen/logrus" +) + +const ( + inlineDepthRelations = "inline-relations-depth" + depth = "2" + listAppsPath = "/v2/apps" +) + +// CfCollectAppsMetadataKind defines kind of cf collect apps file +const CfCollectAppsMetadataKind types.Kind = "CfCollectApps" + +// CfCollectApps defines definition of cf collect apps file +type CfCollectApps struct { + types.TypeMeta `yaml:",inline"` + Spec CfCollectAppsSpec `yaml:"spec,omitempty"` +} + +// CfCollectAppsSpec stores the app information +type CfCollectAppsSpec struct { + Filters CfCollectFilters `yaml:"filters,omitempty"` + CfCollectApps []CfCollectApp `yaml:"applications,omitempty"` +} + +// CfCollectFilters stores the spaceguid and querydepth to be used to filter while collecting metadata +type CfCollectFilters struct { + SpaceGuid string `yaml:"spaceguid,omitempty"` + QueryDepth string `yaml:"query_depth,omitempty"` +} + +// CfCollectApp defines CfCollectApp information +type CfCollectApp struct { + AppSpec CfAppSpec `yaml:"application"` +} + +// CfAppSpec defines CF app spec +type CfAppSpec struct { + Name string `json:"name"` + Guid string `json:"guid"` +} + +// AppResponse defines app response +type AppResponse struct { + Count int `json:"total_results"` + Pages int `json:"total_pages"` + NextUrl string `json:"next_url"` + Resources []AppResource `json:"resources"` +} + +// AppResource defines app resource +type AppResource struct { + Meta cfclient.Meta `json:"metadata"` + Entity App `json:"entity"` +} + +// CfAppsCollector collects cf runtime applications +type CfAppsCollector struct { +} + +// GetAnnotations returns annotations on which this collector should be invoked +func (c *CfAppsCollector) GetAnnotations() []string { + annotations := []string{"cf", "cloudfoundry"} + return annotations +} + +// setQueryDepth sets the depth to be used in +func setQueryDepth(queryDepth string) url.Values { + query := url.Values{} + if queryDepth == "" { + queryDepth = depth + } + query.Set(inlineDepthRelations, queryDepth) + logrus.Debugf("CF collect query depth = %s", queryDepth) + return query +} + +// listAppsBySpaceGuid collects all CF apps data for given spaceGuid, if no appName/appGuid is present. If appName/appGuid is present, then it updates the path to be used in the listAppsByNameOrGuid func +func listAppsBySpaceGuid(client *cfclient.Client, spaceGuid string, queryDepth string, collectApps []App, numCfCollectApps int) (string, []App) { + query := setQueryDepth(queryDepth) + path := "" + if spaceGuid == "" { + return path, collectApps + } + logrus.Debugf("Detected CF Space guid: %s", spaceGuid) + path = fmt.Sprintf("/v2/spaces/%s/apps", spaceGuid) + if numCfCollectApps == 0 { + apps, err := listApps(client, path, query, -1) // If no CF app is specified in yaml, collect all apps in the provided spaceguid + if err != nil { + logrus.Errorf("Unable to collect the cf apps from the Space guid %s : %s", spaceGuid, err) + } else { + collectApps = append(collectApps, apps...) + } + } + return path, collectApps +} + +func getAppByGuid(client *cfclient.Client, guid string, queryDepth string) (App, error) { + var appResource AppResource + query := setQueryDepth(queryDepth) + requestUrl := getRequestUrl(listAppsPath+"/"+guid, query) // /v2/apps/:appGuid fetches the app with given guid + logrus.Debugf("CF collect queryDepth val = %v", queryDepth) + logrus.Debugf("CF collect request URL = %v", requestUrl) + r := client.NewRequest("GET", requestUrl) + resp, err := client.DoRequest(r) + if err != nil { + return App{}, errors.Wrap(err, "Error requesting apps") + } + defer resp.Body.Close() + resBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return App{}, errors.Wrap(err, "Error reading app response body") + } + err = json.Unmarshal(resBody, &appResource) + if err != nil { + return App{}, errors.Wrap(err, "Error unmarshalling app") + } + return mergeAppResource(client, appResource), nil +} + +func getAppsByName(client *cfclient.Client, path string, query url.Values, collectApps []App, appName string) []App { + apps, err := listApps(client, path, query, -1) + if err != nil { + logrus.Errorf("Unable to collect the selected cf app %s : %s", appName, err) + return collectApps + } + if len(apps) != 0 { + collectApps = append(collectApps, apps...) + return collectApps + } + cfErr := cfclient.NewAppNotFoundError() + logrus.Errorf(fmt.Sprintf(cfErr.Description, appName)) + return collectApps +} + +func listAppsByNameOrGuid(client *cfclient.Client, path string, cfCollectApps []CfCollectApp, queryDepth string) []App { + var collectApps []App + for _, cfCollectApp := range cfCollectApps { + if path == "" { + path = listAppsPath + } + appSpec := cfCollectApp.AppSpec + if appSpec.Guid != "" { + app, err := getAppByGuid(client, appSpec.Guid, queryDepth) + if err != nil { + logrus.Errorf("Unable to collect the app with guid %s %q", appSpec.Guid, err) + } else { + collectApps = append(collectApps, app) + continue + } + } + if appSpec.Name != "" { + query := setQueryDepth(queryDepth) + query.Add("q", fmt.Sprintf("name:%s", appSpec.Name)) // /v2/apps/ and /v2/spaces/:spaceGuid/apps/ support querying a particular app by AppName + collectApps = getAppsByName(client, path, query, collectApps, appSpec.Name) + } + } + return collectApps +} + +// collectSelectiveCfApps collects the selected cf apps +func collectSelectiveCfApps(inputPath string, client *cfclient.Client) []App { + var collectApps []App + filePaths, err := common.GetFilesByExt(inputPath, []string{".yml", ".yaml"}) + if err != nil { + logrus.Errorf("failed to look for yaml files in the directory %s . Error: %q", inputPath, err) + return collectApps + } + depth := "" + for _, filePath := range filePaths { + cfInstanceCollectApps := CfCollectApps{} + if err := common.ReadMove2KubeYaml(filePath, &cfInstanceCollectApps); err != nil { + logrus.Debugf("Failed to read the yaml file at path %q Error: %q", filePath, err) + continue + } + if cfInstanceCollectApps.Kind != string(CfCollectAppsMetadataKind) { + logrus.Debugf("%q is not a valid cf collect apps file. Expected kind: %s Actual Kind: %s", filePath, string(CfCollectAppsMetadataKind), cfInstanceCollectApps.Kind) + continue + } + path := "" + if cfInstanceCollectApps.Spec.Filters.QueryDepth != "" { + // This depth var to be used if no apps get collected by name/guid/spaceguid + depth = cfInstanceCollectApps.Spec.Filters.QueryDepth + } + // If the yaml has spaceGuid in the filters and no appName/appGuid is present, collect all CF apps data for given spaceGuid. If appName/appGuid is present, then listAppsBySpaceGuid() updates the path to be used in the listAppsByNameOrGuid() + path, collectAppsBySpaceGuid := listAppsBySpaceGuid(client, cfInstanceCollectApps.Spec.Filters.SpaceGuid, cfInstanceCollectApps.Spec.Filters.QueryDepth, collectApps, len(cfInstanceCollectApps.Spec.CfCollectApps)) + collectApps = append(collectApps, collectAppsBySpaceGuid...) + if len(collectApps) == 0 { + collectApps = append(collectApps, listAppsByNameOrGuid(client, path, cfInstanceCollectApps.Spec.CfCollectApps, cfInstanceCollectApps.Spec.Filters.QueryDepth)...) + } + } + // If the yaml only has query_depth field, and no appGuid, appName, or spaceGuid, then collect all CF apps info for the given query depth + if len(collectApps) == 0 && depth != "" { + collectApps = collectAllCfApps(client, depth) + } + return collectApps +} + +// collectAllCfApps collects all the cf apps +func collectAllCfApps(client *cfclient.Client, depth string) []App { + var collectApps []App + query := setQueryDepth(depth) + apps, err := listApps(client, listAppsPath, query, -1) + if err != nil { + logrus.Errorf("Unable to get list of cf apps : %s", err) + return collectApps + } + collectApps = append(collectApps, apps...) + return collectApps +} + +// Collect gets the cf app metadata by querying the cf app. Assumes that the authentication with cluster is already done. +func (c *CfAppsCollector) Collect(inputPath string, outputPath string) error { + client, err := getCfClient() + if err != nil { + logrus.Errorf("Unable to connect to cf client : %s", err) + return err + } + cfInfo, err := client.GetInfo() + if err != nil { + logrus.Errorf("Unable to get info of cf instance : %s", err) + } + var collectApps []App + if inputPath != "" { + collectApps = collectSelectiveCfApps(inputPath, client) + } else { + collectApps = collectAllCfApps(client, depth) + } + outputPath = filepath.Join(outputPath, "cf") + err = os.MkdirAll(outputPath, common.DefaultDirectoryPermission) + if err != nil { + logrus.Errorf("Unable to create outputPath %s : %s", outputPath, err) + } + cfinstanceapps := NewCfApps() + cfinstanceapps.Name = common.NormalizeForMetadataName(strings.TrimSpace(cfInfo.Name)) + for _, app := range collectApps { + cfapp := CfApp{ + Application: app, + } + appEnv, err := client.GetAppEnv(app.Guid) + if err != nil { + logrus.Errorf("Unable to get app environment data : %s", err) + } else { + cfapp.Environment = appEnv + } + cfinstanceapps.Spec.CfApps = append(cfinstanceapps.Spec.CfApps, cfapp) + } + cfinstanceapps = FormatMapsWithInterface(cfinstanceapps) + fileName := "cfapps-" + cfinstanceapps.Name + if fileName != "" { + outputPath = filepath.Join(outputPath, common.NormalizeForFilename(fileName)+".yaml") + err = common.WriteYaml(outputPath, cfinstanceapps) + if err != nil { + logrus.Errorf("Unable to write collect output : %s", err) + } + return err + } + + return nil +} + +func getRequestUrl(path string, query url.Values) string { + encodedQuery := "" + if query.Encode() != "" { + encodedQuery = "?" + query.Encode() + } + return path + encodedQuery +} + +func listApps(client *cfclient.Client, path string, query url.Values, totalPages int) ([]App, error) { + requestUrl := getRequestUrl(path, query) + logrus.Debugf("CF collect requestUrl = %s", requestUrl) + pages := 0 + apps := []App{} + for { + var appResp AppResponse + r := client.NewRequest("GET", requestUrl) + resp, err := client.DoRequest(r) + + if err != nil { + return nil, errors.Wrap(err, "Error requesting apps") + } + defer resp.Body.Close() + resBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "Error reading app request") + } + + err = json.Unmarshal(resBody, &appResp) + if err != nil { + return nil, errors.Wrap(err, "Error unmarshalling app") + } + + for _, app := range appResp.Resources { + apps = append(apps, mergeAppResource(client, app)) + } + + requestUrl = appResp.NextUrl + if requestUrl == "" || query.Get("page") != "" { + break + } + + pages++ + if totalPages > 0 && pages >= totalPages { + break + } + } + return apps, nil +} + +func mergeAppResource(client *cfclient.Client, app AppResource) App { + app.Entity.Guid = app.Meta.Guid + app.Entity.CreatedAt = app.Meta.CreatedAt + app.Entity.UpdatedAt = app.Meta.UpdatedAt + app.Entity.SpaceData.Entity.Guid = app.Entity.SpaceData.Meta.Guid + app.Entity.SpaceData.Entity.OrgData.Entity.Guid = app.Entity.SpaceData.Entity.OrgData.Meta.Guid + app.Entity.c = client + return app.Entity +} diff --git a/collector/cfservicescollector.go b/collector/cfservicescollector.go new file mode 100644 index 0000000..4cecb5e --- /dev/null +++ b/collector/cfservicescollector.go @@ -0,0 +1,74 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "os" + "path/filepath" + "strings" + + "github.com/konveyor/move2kube-wasm/common" + collecttypes "github.com/konveyor/move2kube-wasm/types/collection" + + "github.com/sirupsen/logrus" +) + +// CfServicesCollector collects cf runtime applications +type CfServicesCollector struct { +} + +// GetAnnotations returns annotations on which this collector should be invoked +func (c *CfServicesCollector) GetAnnotations() []string { + annotations := []string{"cf", "cloudfoundry"} + return annotations +} + +// Collect gets the cf service metadata by querying the cf app. Assumes that the authentication with cluster is already done. +func (c *CfServicesCollector) Collect(inputPath string, outputPath string) error { + client, err := getCfClient() + if err != nil { + logrus.Errorf("Unable to connect to cf client : %s", err) + return err + } + cfInfo, err := client.GetInfo() + if err != nil { + logrus.Errorf("Unable to get info of cf instance : %s", err) + } + services, err := client.ListServices() + if err != nil { + logrus.Errorf("Unable to get list of cf apps : %s", err) + } + outputPath = filepath.Join(outputPath, "cf") + err = os.MkdirAll(outputPath, common.DefaultDirectoryPermission) + if err != nil { + logrus.Errorf("Unable to create outputPath %s : %s", outputPath, err) + } + cfservices := collecttypes.NewCfServices() + cfservices.Name = common.NormalizeForMetadataName(strings.TrimSpace(cfInfo.Name)) + cfservices.Spec.CfServices = services + fileName := "cfservices-" + cfservices.Name + if fileName != "" { + outputPath = filepath.Join(outputPath, common.NormalizeForFilename(fileName)+".yaml") + err = common.WriteYaml(outputPath, cfservices) + if err != nil { + logrus.Errorf("Unable to write collect output : %s", err) + } + return err + } + + return nil +} diff --git a/collector/cfutils.go b/collector/cfutils.go new file mode 100644 index 0000000..5d2eadb --- /dev/null +++ b/collector/cfutils.go @@ -0,0 +1,152 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "os" + "path/filepath" + + cfclient "github.com/cloudfoundry-community/go-cfclient/v2" + "github.com/konveyor/move2kube-wasm/common" + "github.com/konveyor/move2kube-wasm/types" + "github.com/sirupsen/logrus" +) + +// CfAppsMetadataKind defines kind of cf runtime instance apps file +const CfAppsMetadataKind types.Kind = "CfApps" + +// CfApps defines definition of cf runtime instance apps file +type CfApps struct { + types.TypeMeta `yaml:",inline"` + types.ObjectMeta `yaml:"metadata,omitempty"` + Spec CfAppsSpec `yaml:"spec,omitempty"` +} + +// CfAppsSpec stores the data +type CfAppsSpec struct { + CfApps []CfApp `yaml:"applications"` +} + +// CfApp defines CfApp and environment information +type CfApp struct { + Application App `yaml:"application"` + Environment cfclient.AppEnv `yaml:"environment"` +} + +// App defines CF application information +type App struct { + Guid string `json:"guid"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + Name string `json:"name"` + Memory int `json:"memory"` + Instances int `json:"instances"` + DiskQuota int `json:"disk_quota"` + SpaceGuid string `json:"space_guid"` + StackGuid string `json:"stack_guid"` + State string `json:"state"` + PackageState string `json:"package_state"` + Command string `json:"command"` + Buildpack string `json:"buildpack"` + DetectedBuildpack string `json:"detected_buildpack"` + DetectedBuildpackGuid string `json:"detected_buildpack_guid"` + HealthCheckHttpEndpoint string `json:"health_check_http_endpoint"` + HealthCheckType string `json:"health_check_type"` + HealthCheckTimeout int `json:"health_check_timeout"` + Diego bool `json:"diego"` + EnableSSH bool `json:"enable_ssh"` + DetectedStartCommand string `json:"detected_start_command"` + DockerImage string `json:"docker_image"` + DockerCredentialsJSON map[string]interface{} `json:"docker_credentials_json"` + DockerCredentials cfclient.DockerCredentials `json:"docker_credentials"` + Environment map[string]interface{} `json:"environment_json"` + StagingFailedReason string `json:"staging_failed_reason"` + StagingFailedDescription string `json:"staging_failed_description"` + Ports []int `json:"ports"` + SpaceURL string `json:"space_url"` + SpaceData cfclient.SpaceResource `json:"space"` + PackageUpdatedAt string `json:"package_updated_at"` + c *cfclient.Client +} + +// getClient returns a new client for the given cf home directory +func getClient(cfHomeDir string) (*cfclient.Client, error) { + var cfClientConfig *cfclient.Config + var err error + homeDir, err := os.UserHomeDir() + if err != nil { + logrus.Errorf("Error while getting current user's home directory: %s", err) + return nil, err + } + if cfHomeDir == "" { + cfClientConfig, err = cfclient.NewConfigFromCF() + } else { + cfClientConfig, err = cfclient.NewConfigFromCFHome(filepath.Join(homeDir, cfHomeDir)) + } + if err != nil { + logrus.Debugf("Unable to get the cf config: %s", err) + return nil, err + } + client, err := cfclient.NewClient(cfClientConfig) + if err != nil { + if cfHomeDir == "" { + logrus.Debugf("Failed to create a new client using the config.json in .cf directory.") + } else { + logrus.Debugf("Failed to create a new client using the config.json in %s directory.", filepath.Join(homeDir, cfHomeDir, ".cf")) + } + } + return client, err +} + +// getCfClient returns a new cf client +func getCfClient() (*cfclient.Client, error) { + var client *cfclient.Client + var err error + cfHomeDirs := [3]string{"", ".ibmcloud", ".bluemix"} + for _, cfHomeDir := range cfHomeDirs { + client, err = getClient(cfHomeDir) + if err == nil { + break + } + } + return client, err +} + +// FormatMapsWithInterface stringifies interfaces in cloud foundry data +func FormatMapsWithInterface(cfAppInstances CfApps) CfApps { + for index, app := range cfAppInstances.Spec.CfApps { + app.Application.DockerCredentialsJSON = common.JsonifyMapValues(app.Application.DockerCredentialsJSON) + app.Application.Environment = common.JsonifyMapValues(app.Application.Environment) + app.Environment.Environment = common.JsonifyMapValues(app.Environment.Environment) + app.Environment.ApplicationEnv = common.JsonifyMapValues(app.Environment.ApplicationEnv) + app.Environment.RunningEnv = common.JsonifyMapValues(app.Environment.RunningEnv) + app.Environment.StagingEnv = common.JsonifyMapValues(app.Environment.StagingEnv) + app.Environment.SystemEnv = common.JsonifyMapValues(app.Environment.SystemEnv) + cfAppInstances.Spec.CfApps[index] = app + } + return cfAppInstances +} + +// NewCfApps creates a new instance of CfInstanceApps +func NewCfApps() CfApps { + return CfApps{ + TypeMeta: types.TypeMeta{ + Kind: string(CfAppsMetadataKind), + APIVersion: types.SchemeGroupVersion.String(), + }, + } +} diff --git a/collector/clustercollector.go b/collector/clustercollector.go new file mode 100644 index 0000000..227dc61 --- /dev/null +++ b/collector/clustercollector.go @@ -0,0 +1,649 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime/debug" + "sort" + "strings" + + semver "github.com/Masterminds/semver/v3" + "github.com/konveyor/move2kube-wasm/common" + //"github.com/konveyor/move2kube-wasm/transformer/kubernetes/k8sschema" + collecttypes "github.com/konveyor/move2kube-wasm/types/collection" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/runtime/schema" + cgdiscovery "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" // See issue https://github.com/kubernetes/client-go/issues/345 + cgclientcmd "k8s.io/client-go/tools/clientcmd" +) + +// ClusterCollector Implements Collector interface +type ClusterCollector struct { + clusterCmd string +} + +// GetAnnotations returns annotations on which this collector should be invoked +func (c ClusterCollector) GetAnnotations() []string { + return []string{"k8s"} +} + +// Collect gets the cluster metadata by querying the cluster. Assumes that the authentication with cluster is already done. +func (c *ClusterCollector) Collect(inputPath string, outputPath string) error { + //Creating the output sub-directory if it does not exist + outputPath = filepath.Join(outputPath, "clusters") + err := os.MkdirAll(outputPath, common.DefaultDirectoryPermission) + if err != nil { + logrus.Errorf("Unable to create output directory at path %q Error: %q", outputPath, err) + return err + } + cmd := c.getClusterCommand() + if cmd == "" { + errStr := "no kubectl or oc in path. Add kubectl to path and rerun to collect data about the cluster in context" + logrus.Warnf(errStr) + return fmt.Errorf(errStr) + } + name, err := c.getClusterContextName() + if err != nil { + logrus.Warnf("Unable to access cluster in context : %s", err) + return err + } + clusterMd := collecttypes.NewClusterMetadata(name) + if clusterMd.Spec.StorageClasses, err = c.getStorageClasses(); err != nil { + //If no storage classes, this will be an empty array + clusterMd.Spec.StorageClasses = []string{} + } + + clusterMd.Spec.APIKindVersionMap, err = c.collectUsingAPI() + if err != nil { + logrus.Warnf("Failed to collect using the API. Error: %q . Falling back to using the CLI.", err) + clusterMd.Spec.APIKindVersionMap, err = c.collectUsingCLI() + if err != nil { + logrus.Warnf("Failed to collect using the CLI. Error: %q", err) + return err + } + } + + c.groupOrderPolicy(&clusterMd.Spec.APIKindVersionMap) + //c.VersionOrderPolicy(&clusterMd.APIKindVersionMap) + + outputPath = filepath.Join(outputPath, common.NormalizeForFilename(clusterMd.Name)+".yaml") + return common.WriteYaml(outputPath, clusterMd) +} + +func (c *ClusterCollector) getClusterCommand() string { + if c.clusterCmd != "" { + return c.clusterCmd + } + + cmd := "kubectl" + _, err := exec.LookPath(cmd) + if err == nil { + c.clusterCmd = cmd + return c.clusterCmd + } + logrus.Warnf("Unable to find the %s command. Error: %q", cmd, err) + + cmd = "oc" + _, err = exec.LookPath(cmd) + if err == nil { + c.clusterCmd = cmd + return c.clusterCmd + } + logrus.Warnf("Unable to find the %s command. Error: %q", cmd, err) + + return "" +} + +func (c *ClusterCollector) getClusterContextName() (string, error) { + cmd := exec.Command(c.getClusterCommand(), "config", "current-context") + name, err := cmd.Output() + return strings.TrimSpace(string(name)), err +} + +func (c *ClusterCollector) getStorageClasses() ([]string, error) { + ccmd := c.getClusterCommand() + cmd := exec.Command(ccmd, "get", "sc", "-o", "yaml") + yamlOutput, err := cmd.CombinedOutput() + if err != nil { + errDesc := c.interpretError(string(yamlOutput)) + if errDesc != "" { + logrus.Warnf("Error while running %s. %s", ccmd, errDesc) + } else { + logrus.Warnf("Error while fetching storage classes using command [%s]", cmd) + } + return nil, err + } + + fileContents := map[string]interface{}{} + err = yaml.Unmarshal(yamlOutput, &fileContents) + if err != nil { + logrus.Errorf("Error in unmarshalling yaml: %s. Skipping.", err) + return nil, err + } + + scArray := fileContents["items"].([]interface{}) + storageClasses := []string{} + + for _, sc := range scArray { + if mapSC, ok := sc.(map[string]interface{}); ok { + storageClasses = append(storageClasses, mapSC["metadata"].(map[string]interface{})["name"].(string)) + } else { + logrus.Warnf("Unknown type detected in cluster metadata [%T]", mapSC) + } + } + + return storageClasses, nil +} + +func (c *ClusterCollector) interpretError(cmdOutput string) string { + errorTerms := []string{"Unauthorized", "Username"} + + for _, e := range errorTerms { + if c.getClusterCommand() == "oc" && strings.Contains(cmdOutput, e) { + return "Please login to cluster before running collect. (e.g. oc login --token=)" + } else if c.getClusterCommand() == "kubectl" && strings.Contains(cmdOutput, e) { + return "Please configure the cluster authentication with following instructions: [https://kubernetes.io/docs/reference/kubectl/cheatsheet/#kubectl-context-and-configuration]" + } + } + + return "" +} + +func (c ClusterCollector) getGlobalGroupOrder() []string { + return []string{`^.+\.k8s\.io$`, `^apps$`, `^policy$`, `^extensions$`, `^.+\.openshift\.io$`} +} + +func (c *ClusterCollector) getAPI() (*cgdiscovery.DiscoveryClient, error) { + rules := cgclientcmd.NewDefaultClientConfigLoadingRules() + cfg, err := cgclientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, &cgclientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + logrus.Warnf("Failed to get the default config for the cluster API client. Error: %q", err) + return nil, err + } + return cgdiscovery.NewDiscoveryClientForConfig(cfg) +} + +func (c *ClusterCollector) getPreferredResourceUsingAPI(api *cgdiscovery.DiscoveryClient) ([]schema.GroupVersion, error) { + defer func() []schema.GroupVersion { + if rErr := recover(); rErr != nil { + logrus.Errorf("Recovered from error in getPreferredResourceUsingAPI [%s]", rErr) + return nil + } + return []schema.GroupVersion{} + }() + debug.SetPanicOnFault(true) + var gvList []schema.GroupVersion + if api == nil { + logrus.Errorf("API object is null") + return nil, fmt.Errorf("API object is null") + } + apiGroupList, err := api.ServerGroups() + if err != nil { + logrus.Errorf("API request for server-group list failed") + return nil, err + } + for _, group := range apiGroupList.Groups { + preferredGV, err := schema.ParseGroupVersion(group.PreferredVersion.GroupVersion) + if err != nil { + continue + } + gvList = append(gvList, preferredGV) + prioritizedGVList := scheme.Scheme.PrioritizedVersionsForGroup(group.Name) + for _, prioritizedGV := range prioritizedGVList { + if strings.Compare(group.PreferredVersion.GroupVersion, prioritizedGV.String()) == 0 { + continue + } + for _, gvObj := range group.Versions { + if strings.Compare(prioritizedGV.String(), gvObj.GroupVersion) == 0 { + gv, _ := schema.ParseGroupVersion(gvObj.GroupVersion) + gvList = append(gvList, gv) + break + } + } + } + for _, version := range group.Versions { + gv, _ := schema.ParseGroupVersion(version.GroupVersion) + if gvExists(gvList, gv) { + continue + } + gvList = append(gvList, gv) + } + } + return gvList, nil +} + +func (c *ClusterCollector) getKindsForGroups(api *cgdiscovery.DiscoveryClient) (map[string][]schema.GroupVersion, error) { + defer func() map[string][]schema.GroupVersion { + if rErr := recover(); rErr != nil { + logrus.Errorf("Recovered from error in getKindsForGroups [%s]", rErr) + return nil + } + var emptyMap map[string][]schema.GroupVersion + return emptyMap + }() + + mapKind := map[string][]schema.GroupVersion{} + + _, apiResourceList, err := api.ServerGroupsAndResources() + if err != nil { + return nil, err + } + + for _, rscListObj := range apiResourceList { + gvObj, err := schema.ParseGroupVersion(rscListObj.GroupVersion) + for err != nil { + logrus.Warnf("Ignoring group-version [%s]. Could not parse it", rscListObj.GroupVersion) + continue + } + + for _, rscObj := range rscListObj.APIResources { + if gvList, ok := mapKind[rscObj.Kind]; ok { + if !gvExists(gvList, gvObj) { + gvList = append(gvList, gvObj) + } + mapKind[rscObj.Kind] = gvList + } else { + mapKind[rscObj.Kind] = []schema.GroupVersion{gvObj} + + } + } + } + + return mapKind, nil +} + +func (c *ClusterCollector) sortGroupVersionByPreference(prefGVList []schema.GroupVersion, mapKind *map[string][]schema.GroupVersion) { + for kind, gvList := range *mapKind { + var gvOrderedList []schema.GroupVersion + unorderedList := []string{} + for _, pGV := range prefGVList { + if gvExists(gvList, pGV) { + gvOrderedList = append(gvOrderedList, pGV) + } else { + unorderedList = append(unorderedList, pGV.String()) + } + } + + unorderedList = c.clusterByGroupsAndSortVersions(unorderedList) + for _, gvStr := range unorderedList { + gvObj, err := schema.ParseGroupVersion(gvStr) + if err == nil { + continue + } + gvOrderedList = append(gvOrderedList, gvObj) + } + (*mapKind)[kind] = gvOrderedList + } +} + +func (c *ClusterCollector) collectUsingAPI() (map[string][]string, error) { + api, err := c.getAPI() + if err != nil { + logrus.Warnf("Failed to api handle for cluster") + return nil, err + } + + gvList, err := c.getPreferredResourceUsingAPI(api) + errStr := "Failed to retrieve preferred group information from cluster" + if err != nil { + logrus.Warnf(errStr) + return nil, err + } else if len(gvList) == 0 { + logrus.Warnf(errStr) + return nil, fmt.Errorf(errStr) + } + + mapKind, err := c.getKindsForGroups(api) + errStr = "Failed to retrieve information from cluster" + if err != nil { + logrus.Warnf(errStr) + return nil, err + } else if len(mapKind) == 0 { + logrus.Warnf(errStr) + return nil, fmt.Errorf(errStr) + } + + c.sortGroupVersionByPreference(gvList, &mapKind) + + APIKindVersionMap := map[string][]string{} + + for kind, gvList := range mapKind { + gvStrList := make([]string, len(gvList)) + for i, gv := range gvList { + gvStrList[i] = gv.String() + } + APIKindVersionMap[kind] = gvStrList + } + + return APIKindVersionMap, nil +} + +func (c *ClusterCollector) getAllGVMatchingGroup(groupRegex string, gvList []string) []string { + var filtered []string + + for _, gv := range gvList { + gvObj, err := schema.ParseGroupVersion(gv) + if err != nil { + continue + } + + if gvObj.Group == "" { + continue + } + + re := regexp.MustCompile(groupRegex) + if re.MatchString(gvObj.Group) { + filtered = append(filtered, gv) + } + } + + return filtered +} + +func (c *ClusterCollector) groupOrderPolicy(mapKindGV *map[string][]string) { + globalOrder := c.getGlobalGroupOrder() + for kind, gvList := range *mapKindGV { + sortedGV := []string{} + + //First priority is for known groups + for _, groupKey := range globalOrder { + subsetOfGV := c.getAllGVMatchingGroup(groupKey, gvList) + sortedGV = append(sortedGV, subsetOfGV...) + } + + //Second priority is for unknown groups (which are not empty string) + for _, gv := range gvList { + gvObj, err := schema.ParseGroupVersion(gv) + if err != nil { + continue + } + + if common.IsPresent(sortedGV, gv) { + continue + } + + if strings.Compare(gvObj.Group, "") != 0 { + sortedGV = append(sortedGV, gv) + } + } + + //Third priority is for empty groups + for _, gv := range gvList { + gvObj, err := schema.ParseGroupVersion(gv) + if err != nil { + continue + } + + if strings.Compare(gvObj.Group, "") == 0 { + sortedGV = append(sortedGV, gv) + } + } + + if len(sortedGV) > 0 { + (*mapKindGV)[kind] = sortedGV + } else { + (*mapKindGV)[kind] = gvList + } + } +} + +func (c *ClusterCollector) sortVersionList(vList *[]string) { + srcVersionKeys := []string{"alpha", "beta"} + trVersionKeys := []string{"-alpha.", "-beta."} + regex := []string{`\-alpha\.`, `\-beta\.`} + + for index, version := range *vList { + for i, vKey := range srcVersionKeys { + re := regexp.MustCompile(vKey) + if re.MatchString(version) { + //Tranforming the string to the format suitable for semver pkg + (*vList)[index] = re.ReplaceAllString(version, trVersionKeys[i]) + break + } + } + } + + svObjList := make([]*semver.Version, len(*vList)) + for index, versionStr := range *vList { + svObj, err := semver.NewVersion(versionStr) + if err != nil { + logrus.Warnf("Skipping Version: %s", versionStr) + continue + } + + svObjList[index] = svObj + } + + sort.Sort(sort.Reverse(semver.Collection(svObjList))) + + for index, svObj := range svObjList { + transfVersionStr := svObj.Original() + noMatches := true + for i, vKey := range regex { + re := regexp.MustCompile(vKey) + if re.MatchString(transfVersionStr) { + (*vList)[index] = re.ReplaceAllString(transfVersionStr, srcVersionKeys[i]) + noMatches = false + break + } + } + if noMatches { + (*vList)[index] = transfVersionStr + } + } +} + +func (c *ClusterCollector) clusterByGroupsAndSortVersions(gvList []string) []string { + gvMap := map[string][]string{} + for _, gvStr := range gvList { + gvObj, err := schema.ParseGroupVersion(gvStr) + if err != nil { + logrus.Debugf("Error parting group version [%s]", gvStr) + continue + } + + if vList, ok := gvMap[gvObj.Group]; ok { + vList = append(vList, gvObj.Version) + gvMap[gvObj.Group] = vList + } else { + vList = []string{gvObj.Version} + gvMap[gvObj.Group] = vList + } + } + + for _, vList := range gvMap { + c.sortVersionList(&vList) + } + + sortedGVList := []string{} + for group, vList := range gvMap { + for _, v := range vList { + gvObj := schema.GroupVersion{Group: group, Version: v} + sortedGVList = append(sortedGVList, gvObj.String()) + } + } + + return sortedGVList +} + +func (c *ClusterCollector) collectUsingCLI() (map[string][]string, error) { + cmd := exec.Command("bash", "-c", c.getClusterCommand()+" api-resources -o name") + output, err := cmd.Output() + if err != nil { + logrus.Errorf("Error while running kubectl api-resources: %s", err) + return nil, err + } + logrus.Debugf("Got kind information for cluster") + nameList := strings.Split(string(output), "\n") + mapKind := map[string][]schema.GroupVersion{} + for _, name := range nameList { + tmpArray := strings.Split(name, ".") + if len(tmpArray) > 0 { + name := tmpArray[0] + kind, gvStr, err := c.getGVKUsingNameCLI(name) + if err != nil { + logrus.Debugf("Erroring parsing kind from CLI output") + continue + } + group := "" + for i, tmp := range tmpArray { + if i == 1 { + group = tmp + } else if i > 1 { + tmp = strings.TrimSpace(tmp) + group = group + "." + tmp + } + } + + if group != "" { + if groupArray, ok := mapKind[kind]; ok { + groupArray = append(groupArray, schema.GroupVersion{Group: group, Version: ""}) + mapKind[kind] = groupArray + } else { + mapKind[kind] = []schema.GroupVersion{{Group: group, Version: ""}} + } + } else { + mapKind[kind] = []schema.GroupVersion{{Group: "", Version: gvStr}} + } + } + } + + apiMd := map[string][]string{} + + for kind, availableGroupList := range mapKind { + if len(availableGroupList) == 1 { + singletonObj := availableGroupList[0] + if strings.Compare(singletonObj.Group, "") == 0 { + apiMd[kind] = []string{singletonObj.Version} + continue + } + } + if len(availableGroupList) > 0 { + gvList := c.getPreferredGVUsingCLI(kind, availableGroupList) + apiMd[kind] = gvList + } else { + logrus.Warnf("Empty group for kind [%s]", kind) + } + } + + return apiMd, nil +} + +func (c *ClusterCollector) getPreferredGVUsingCLI(kind string, availableGroupList []schema.GroupVersion) []string { + //TODO: WASI + //scheme := k8sschema.GetSchema() + //var gvList []string + //for _, gvObj := range availableGroupList { + // prioritizedGVList := scheme.PrioritizedVersionsForGroup(gvObj.Group) + // if len(prioritizedGVList) > 0 { + // for _, gv := range prioritizedGVList { + // isSupported, err := c.isSupportedGV(kind, gv.String()) + // if isSupported { + // gvList = append(gvList, gv.String()) + // } else { + // logrus.Debugf("Group version not found by CLI for kind [%s] : %s", kind, err) + // } + // } + // } else { + // _, gvStr, err := c.getGVKUsingNameCLI(kind) + // if err == nil { + // gvList = append(gvList, gvStr) + // } + // } + //} + // + //return gvList + return nil +} + +func (c *ClusterCollector) isSupportedGV(kind string, gvStr string) (bool, error) { + cmd := exec.Command("bash", "-c", c.getClusterCommand()+" explain "+kind+" --api-version="+gvStr+" --recursive") + output, err := cmd.Output() + if err != nil { + logrus.Debugf("Error while running %s for verifying [%s]\n", c.getClusterCommand(), gvStr) + return false, err + } + + lines := strings.Split(string(output), "\n") + + if len(lines) < 2 { + return false, fmt.Errorf("description incomplete") + } + + if strings.Contains(lines[1], "VERSION") { + return true, nil + } + + return false, fmt.Errorf("GV [%s] not found", gvStr) +} + +func (c *ClusterCollector) getGVKUsingNameCLI(name string) (string, string, error) { + cmd := exec.Command("bash", "-c", c.getClusterCommand()+" explain "+name) + output, err := cmd.Output() + if err != nil { + //logrus.Errorf("Error while running kubectl: %s\n", err) + return "", "", err + } + + var gvk schema.GroupVersionKind + + lines := strings.Split(string(output), "\n") + + if len(lines) < 2 { + return "", "", fmt.Errorf("description incomplete") + } + + if strings.Contains(lines[0], "KIND") { + tmpArray := strings.Split(lines[0], ":") + gvk.Kind = strings.TrimSpace(tmpArray[1]) + } else { + return "", "", err + } + + if strings.Contains(lines[1], "VERSION") { + tmpArray := strings.Split(lines[1], ":") + tmpGV := strings.TrimSpace(tmpArray[1]) + tmpGVLines := strings.Split(tmpGV, "/") + if len(tmpGVLines) == 2 { + gvk.Group = tmpGVLines[0] + gvk.Version = tmpGVLines[1] + } else { + gvk.Group = "" + gvk.Version = tmpGVLines[0] + } + } + + return gvk.Kind, gvk.GroupVersion().String(), nil +} + +// GVExists looks up group version from list +func gvExists(gvList []schema.GroupVersion, gvKey schema.GroupVersion) bool { + for _, gv := range gvList { + if gv.String() == gvKey.String() { + return true + } + } + return false +} diff --git a/collector/collector.go b/collector/collector.go new file mode 100644 index 0000000..4e17b0c --- /dev/null +++ b/collector/collector.go @@ -0,0 +1,29 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +//Collector defines interface for collecting data from data sources +type Collector interface { + Collect(inputDirectory string, outputPath string) error + GetAnnotations() []string +} + +// GetCollectors returns different collectors +func GetCollectors() ([]Collector, error) { + collectors := []Collector{new(ClusterCollector), new(ImagesCollector), new(CfAppsCollector), new(CfServicesCollector)} + return collectors, nil +} diff --git a/collector/imagescollector.go b/collector/imagescollector.go new file mode 100644 index 0000000..1157d40 --- /dev/null +++ b/collector/imagescollector.go @@ -0,0 +1,178 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package collector + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + sourcetypes "github.com/konveyor/move2kube-wasm/collector/sourcetypes" + "github.com/konveyor/move2kube-wasm/common" + collecttypes "github.com/konveyor/move2kube-wasm/types/collection" + "github.com/sirupsen/logrus" + "github.com/spf13/cast" +) + +// ImagesCollector collects the docker images +type ImagesCollector struct { +} + +// GetAnnotations returns annotations on which this collector should be invoked +func (c ImagesCollector) GetAnnotations() []string { + annotations := []string{"k8s", "dockerswarm", "dockercompose"} + return annotations +} + +// Collect gets the image metadata using docker inspect +func (c *ImagesCollector) Collect(inputDirectory string, outputPath string) error { + //Creating the output sub-directory if it does not exist + outputPath = filepath.Join(outputPath, "images") + err := os.MkdirAll(outputPath, common.DefaultDirectoryPermission) + if err != nil { + logrus.Errorf("Unable to create output directory %s : %s", outputPath, err) + return err + } + imageNames, err := getImageNames(inputDirectory) + if err != nil { + return err + } + logrus.Debugf("Images : %s", imageNames) + for _, imageName := range imageNames { + imagedata, err := getDockerInspectResult(imageName) + if err != nil { + continue + } + if imagedata != nil { + imageInfo := getImageInfo(imagedata) + shortesttag := "" + for _, tag := range imageInfo.Spec.Tags { + if shortesttag == "" { + shortesttag = tag + } else { + if len(shortesttag) > len(tag) { + shortesttag = tag + } + } + } + imagefile := filepath.Join(outputPath, common.NormalizeForFilename(shortesttag)+".yaml") + err := common.WriteYaml(imagefile, imageInfo) + if err != nil { + logrus.Errorf("Unable to write file %s : %s", imagefile, err) + } + } + } + + return nil +} + +func getDockerInspectResult(imageName string) ([]byte, error) { + cmd := exec.Command("docker", "inspect", imageName) + jsonOutput, err := cmd.CombinedOutput() + if err != nil { + if strings.Contains(string(jsonOutput), "permission denied") { + logrus.Warnf("Error while running docker-inspect due to lack of permissions") + logrus.Warnf("Please refer to [https://docs.docker.com/engine/install/linux-postinstall/] to fix this issue") + } else if strings.Contains(string(jsonOutput), "No such object") { + logrus.Warnf("Image [%s] not available in local image repo. Run \"docker pull %s\"", imageName, imageName) + return nil, nil + } else { + logrus.Warnf("Error while running docker-inspect: %s", err) + } + return nil, err + } + return jsonOutput, nil +} + +func getImageInfo(data []byte) collecttypes.ImageInfo { + imageInfo := collecttypes.NewImageInfo() + imgLayerInfo := []sourcetypes.DockerImage{} + err := json.Unmarshal(data, &imgLayerInfo) + if err != nil { + logrus.Errorf("Unable to unmarshal image info : %s", err) + } + for _, image := range imgLayerInfo { + imageInfo.Spec.Tags = image.RepoTags + imageInfo.Spec.UserID, err = cast.ToIntE(image.CConfig.User) + if err != nil { + logrus.Debugf("UserID not available in image metadata for [%s]", image.RepoTags[0]) + imageInfo.Spec.UserID = -1 + } + imageInfo.Spec.AccessedDirs = append(imageInfo.Spec.AccessedDirs, image.CConfig.WorkingDir) + for key := range image.CConfig.EPorts { + regex := regexp.MustCompile("[0-9]+") + portNumber, err := cast.ToInt32E(string(regex.FindAll([]byte(key), -1)[0])) + if err != nil { + logrus.Debugf("PortNumber not available in image metadata for [%s]", image.RepoTags[0]) + } else { + imageInfo.Spec.PortsToExpose = append(imageInfo.Spec.PortsToExpose, portNumber) + } + } + } + return imageInfo +} + +func getImageNames(inputPath string) ([]string, error) { + if inputPath == "" { + return getAllImageNames() + } + return getDCImageNames(inputPath) +} + +func getAllImageNames() ([]string, error) { + cmd := exec.Command("bash", "-c", "docker image list --format '{{.Repository}}:{{.Tag}}'") + outputStr, err := cmd.Output() + if err != nil { + logrus.Warnf("Error while running docker image list : %s", err) + return nil, err + } + if len(outputStr) == 0 { + return []string{}, err + } + cleanimages := []string{} + images := strings.Split(strings.TrimSpace(string(outputStr)), "\n") + for _, image := range images { + if strings.HasPrefix(image, "") || strings.HasSuffix(image, "") { + logrus.Debugf("Ignore image with : %s", image) + continue + } else { + cleanimages = append(cleanimages, image) + } + } + logrus.Debugf("clean images : %s", cleanimages) + return cleanimages, err +} + +func getDCImageNames(directorypath string) ([]string, error) { + var imageNames []string + files, err := common.GetFilesByExt(directorypath, []string{".yml", ".yaml"}) + if err != nil { + logrus.Warnf("Unable to fetch yaml files and recognize Docker image yamls : %s", err) + } + for _, path := range files { + dc := sourcetypes.DockerCompose{} + if common.ReadYaml(path, &dc) == nil { + for _, dcservice := range dc.DCServices { + imageNames = append(imageNames, dcservice.Image) + } + } + } + return imageNames, nil +} diff --git a/collector/sourcetypes/dockercompose.go b/collector/sourcetypes/dockercompose.go new file mode 100644 index 0000000..9805cca --- /dev/null +++ b/collector/sourcetypes/dockercompose.go @@ -0,0 +1,28 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sourcetypes + +// DockerCompose reads docker compose files +type DockerCompose struct { + Version string `yaml:"version"` + DCServices map[string]DCService `yaml:"services"` +} + +// DCService reads service +type DCService struct { + Image string `yaml:"image,omitempty"` +} diff --git a/collector/sourcetypes/dockerinspect.go b/collector/sourcetypes/dockerinspect.go new file mode 100644 index 0000000..fcf073b --- /dev/null +++ b/collector/sourcetypes/dockerinspect.go @@ -0,0 +1,31 @@ +/* + * Copyright IBM Corporation 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sourcetypes + +// DockerImage loads docker image +type DockerImage struct { + RepoTags []string `json:"RepoTags"` + CConfig ContainerConfig `json:"ContainerConfig"` +} + +// ContainerConfig loads container config +type ContainerConfig struct { + EPorts map[string]interface{} `json:"ExposedPorts"` + User string `json:"User"` + Env []string `json:"Env"` + WorkingDir string `json:"WorkingDir"` +} diff --git a/common/utils.go b/common/utils.go index 08536ab..c3ac5a0 100644 --- a/common/utils.go +++ b/common/utils.go @@ -22,6 +22,7 @@ import ( "compress/gzip" "crypto/sha256" "embed" + "encoding/json" "encoding/xml" "errors" "fmt" @@ -30,11 +31,13 @@ import ( "github.com/konveyor/move2kube-wasm/types" "github.com/mitchellh/mapstructure" "gopkg.in/yaml.v3" + "hash/crc64" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "reflect" + "strconv" "text/template" "io" @@ -814,3 +817,91 @@ func ConvertStringSelectorsToSelectors(transformerSelector string) (labels.Selec } return lblSelector, err } + +// NormalizeForFilename normalizes a string to only filename valid characters +func NormalizeForFilename(name string) string { + processedString := MakeFileNameCompliant(name) + //TODO: Make it more robust by taking some characters from start and also from end + const maxPrefixLength = 200 + if len(processedString) > maxPrefixLength { + processedString = processedString[0:maxPrefixLength] + } + crc64Table := crc64.MakeTable(0xC96C5795D7870F42) + crc64Int := crc64.Checksum([]byte(name), crc64Table) + return processedString + "-" + strconv.FormatUint(crc64Int, 16) +} + +// MakeFileNameCompliant returns a DNS-1123 standard string +// Motivated by https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set +// Also see page 1 "ASSUMPTIONS" heading of https://tools.ietf.org/html/rfc952 +// Also see page 13 of https://tools.ietf.org/html/rfc1123#page-13 +func MakeFileNameCompliant(name string) string { + if name == "" { + logrus.Error("The file name is empty.") + return "" + } + baseName := filepath.Base(name) + invalidChars := regexp.MustCompile("[^a-zA-Z0-9-.]+") + processedName := invalidChars.ReplaceAllLiteralString(baseName, "-") + if len(processedName) > 63 { + logrus.Debugf("Warning: The processed name %q is longer than 63 characters long.", processedName) + } + first := processedName[0] + last := processedName[len(processedName)-1] + if first == '-' || first == '.' || last == '-' || last == '.' { + logrus.Debugf("Warning: The first and/or last characters of the name %q are not alphanumeric.", processedName) + } + return processedName +} + +// ReadYaml reads an yaml into an object +func ReadYaml(file string, data interface{}) error { + yamlFile, err := os.ReadFile(file) + if err != nil { + logrus.Debugf("Error in reading yaml file %s: %s.", file, err) + return err + } + err = yaml.Unmarshal(yamlFile, data) + if err != nil { + logrus.Debugf("Error in unmarshalling yaml file %s: %s.", file, err) + return err + } + rv := reflect.ValueOf(data) + if rv.Kind() == reflect.Ptr && rv.Elem().Kind() == reflect.Struct { + rv = rv.Elem() + fv := rv.FieldByName("APIVersion") + if fv.IsValid() { + if fv.Kind() == reflect.String { + val := strings.TrimSpace(fv.String()) + if strings.HasPrefix(val, types.SchemeGroupVersion.Group) && !strings.HasSuffix(val, types.SchemeGroupVersion.Version) { + logrus.Warnf("The application file (%s) was generated using a different version than (%s)", val, types.SchemeGroupVersion.String()) + } + } + } + } + return nil +} + +// JsonifyMapValues converts the map values to json +func JsonifyMapValues(inputMap map[string]interface{}) map[string]interface{} { + for key, value := range inputMap { + if value == nil { + inputMap[key] = "" + continue + } + if val, ok := value.(string); ok { + inputMap[key] = val + continue + } + var b bytes.Buffer + encoder := json.NewEncoder(&b) + if err := encoder.Encode(value); err != nil { + logrus.Error("Unable to unmarshal data to json while converting map interfaces to string") + continue + } + strValue := b.String() + strValue = strings.TrimSpace(strValue) + inputMap[key] = strValue + } + return inputMap +} \ No newline at end of file diff --git a/go.mod b/go.mod index 8d6a8d6..7a54756 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/mholt/archiver/v3 v3.5.1 github.com/mitchellh/mapstructure v1.5.0 + github.com/pkg/errors v0.9.1 github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.16.0 @@ -20,12 +21,15 @@ require ( golang.org/x/mod v0.10.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.26.5 + k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible ) require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/andybalholm/brotli v1.0.3 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect @@ -37,6 +41,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/gnostic v0.5.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.13 // indirect @@ -54,7 +59,7 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc4 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pierrec/lz4/v4 v4.1.2 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -64,7 +69,9 @@ require ( golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect + golang.org/x/time v0.1.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect @@ -72,6 +79,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools/v3 v3.5.1 // indirect + k8s.io/api v0.23.1 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index d47bd0f..0e920a2 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,9 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -36,6 +39,13 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -66,6 +76,7 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -90,6 +101,8 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -118,6 +131,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -125,6 +139,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -140,6 +155,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -148,6 +164,7 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -156,6 +173,7 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -178,6 +196,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -185,6 +205,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -192,6 +213,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -201,6 +223,7 @@ github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -276,6 +299,7 @@ github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -288,6 +312,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0= github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -314,6 +339,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -337,12 +363,15 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= @@ -413,9 +442,12 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -432,6 +464,9 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -481,7 +516,11 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -494,7 +533,10 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -509,6 +551,9 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -584,6 +629,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -628,7 +675,12 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -645,6 +697,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -696,8 +750,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.23.1 h1:ncu/qfBfUoClqwkTGbeRqqOqBCRoUAflMuOaOD7J0c8= +k8s.io/api v0.23.1/go.mod h1:WfXnOnwSqNtG62Y1CdjoMxh7r7u9QXGCkA1u0na2jgo= k8s.io/apimachinery v0.23.1 h1:sfBjlDFwj2onG0Ijx5C+SrAoeUscPrmghm7wHP+uXlo= k8s.io/apimachinery v0.23.1/go.mod h1:SADt2Kl8/sttJ62RRsi9MIV4o8f5S3coArm0Iu3fBno= +k8s.io/client-go v0.23.1 h1:Ma4Fhf/p07Nmj9yAB1H7UwbFHEBrSPg8lviR24U2GiQ= +k8s.io/client-go v0.23.1/go.mod h1:6QSI8fEuqD4zgFK0xbdwfB/PthBsIxCJMa3s17WlcO0= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= diff --git a/lib/collector.go b/lib/collector.go new file mode 100644 index 0000000..659aa46 --- /dev/null +++ b/lib/collector.go @@ -0,0 +1,63 @@ +/* + * Copyright IBM Corporation 2020, 2021 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package lib + +import ( + "os" + "strings" + + collector "github.com/konveyor/move2kube-wasm/collector" + "github.com/konveyor/move2kube-wasm/common" + "github.com/sirupsen/logrus" +) + +// Collect gets the metadata from multiple sources, filters it and dumps it into files within source directory +func Collect(inputPath string, outputPath string, annotations []string) { + collectors, err := collector.GetCollectors() + if err != nil { + logrus.Fatalf("Failed to get the collectors. Error: %q", err) + } + if err := os.MkdirAll(outputPath, common.DefaultDirectoryPermission); err != nil { + logrus.Fatalf("Unable to create output directory at path %q Error: %q", outputPath, err) + } + logrus.Infoln("Begin collection") + for _, collector := range collectors { + if len(annotations) > 0 { + if !hasOverlap(annotations, collector.GetAnnotations()) { + continue + } + } + logrus.Infof("[%T] Begin collection", collector) + if err = collector.Collect(inputPath, outputPath); err != nil { + logrus.Warnf("[%T] failed. Error: %q", collector, err) + continue + } + logrus.Infof("[%T] Done", collector) + } + logrus.Infoln("Collection done") +} + +func hasOverlap(a []string, b []string) bool { + for _, val1 := range a { + for _, val2 := range b { + if strings.EqualFold(val1, val2) { + return true + } + } + } + return false +}