feat: Download settings (#19)

* refactor: Rename downloader to classic

* feat: Implement outline for settings 2.0 download

* feat: Implement ListSchemas

* feat: Implement ListSettings

* feat: Write downloaded settings

* feat: added unit test for settings::Download

* feat: Add option to skip settings download

* ci: Make mocks before vet

* ci: Set up go for static-code-analysis workflow

* ci: Integration tests target requires mocks

Co-authored-by: David Laubreiter <[email protected]>

.github/workflows/pr-static-code-analysis.yml

@@ -14,9 +14,18 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version: '~1.19'
+        id: go
+
       - uses: actions/checkout@v3
+
       - name: 🕵️ Go vet
         run: make vet
+
       - name: golangci-lint
         uses: reviewdog/action-golangci-lint@v2
         with:

Makefile

@@ -36,7 +36,7 @@ mocks: setup
 	@echo "Generating mocks"
 	@go generate ./...
 
-vet:
+vet: mocks
 	@echo "Vetting files"
 	@go vet ./...
 
@@ -79,19 +79,19 @@ test: setup mocks lint
 	@echo "Testing ${BINARY}..."
 	@gotestsum ${testopts} -- -tags=unit -v ./...
 
-integration-test: setup
+integration-test: mocks
 	@gotestsum ${testopts} --format standard-verbose -- -tags=integration -timeout=30m -v ./...
 
-integration-test-v1:setup
+integration-test-v1:mocks
 	@gotestsum ${testopts} --format standard-verbose -- -tags=integration_v1 -timeout=30m -v ./...
 
-download-restore-test: setup
+download-restore-test: mocks
 	@gotestsum ${testopts} --format standard-verbose -- -tags=download_restore -timeout=30m -v ./...
 
 clean-environments:
 	@gotestsum ${testopts} --format standard-verbose -- -tags=cleanup -v ./...
 
-nightly-test:setup
+nightly-test:mocks
 	@gotestsum ${testopts} --format standard-verbose -- -tags=nightly -timeout=60m -v ./...
 
 # Build and Test a single package supplied via pgk variable, without using test cache

cmd/monaco/download/download.go

@@ -18,7 +18,8 @@ import (
 	"fmt"
 	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/api"
 	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/download"
-	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/download/downloader"
+	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/download/classic"
+	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/download/settings"
 	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/manifest"
 	project "github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/project/v2"
 	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/project/v2/topologysort"
@@ -172,6 +173,7 @@ type downloadOptions struct {
 	apis                   api.ApiMap
 	forceOverwriteManifest bool
 	clientFactory          dynatraceClientFactory
+	skipSettings           bool
 }
 
 func (c downloadOptions) getDynatraceClient() (rest.DynatraceClient, error) {
@@ -243,8 +245,15 @@ func downloadConfigs(context downloadOptions) (project.ConfigsPerType, error) {
 		return nil, fmt.Errorf("failed to create Dynatrace client: %w", err)
 	}
 
-	downloadedConfigs := downloader.DownloadAllConfigs(context.apis, client, context.projectName)
-	return downloadedConfigs, nil
+	configObjects := classic.DownloadAllConfigs(context.apis, client, context.projectName)
+
+	if !context.skipSettings {
+		settingsObjects := settings.Download(client)
+
+		maps.Copy(configObjects, settingsObjects)
+	}
+
+	return configObjects, nil
 }
 
 func sumConfigs(configs project.ConfigsPerType) int {

cmd/monaco/download/download_integration_test.go

@@ -857,6 +857,7 @@ func getTestingDownloadOptions(server *httptest.Server, projectName string, apiM
 		outputFolder:    "out",
 		projectName:     projectName,
 		apis:            apiMap,
+		skipSettings:    true,
 		clientFactory: func(environmentUrl, token string) (rest.DynatraceClient, error) {
 			return rest.NewDynatraceClientForTesting(environmentUrl, token, server.Client())
 		},

pkg/download/downloader/doc.go → pkg/download/classic/doc.go

@@ -48,4 +48,4 @@ The process looks like this:
 
 ```
 */
-package downloader
+package classic

pkg/download/downloader/download_filter.go → pkg/download/classic/download_filter.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package downloader
+package classic
 
 import (
 	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/api"

pkg/download/downloader/download_filter_test.go → pkg/download/classic/download_filter_test.go

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package downloader
+package classic
 
 import (
 	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/api"

pkg/download/downloader/download_sanitize.go → pkg/download/classic/download_sanitize.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package downloader
+package classic
 
 var apiSanitizeFunctions = map[string]func(properties map[string]interface{}) map[string]interface{}{
 	"service-detection-full-web-service":   removeOrderProperty,

pkg/download/downloader/download_sanitize_test.go → pkg/download/classic/download_sanitize_test.go

@@ -14,7 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package downloader
+package classic
 
 import (
 	"gotest.tools/assert"

pkg/download/downloader/downloader.go → pkg/download/classic/downloader.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package downloader
+package classic
 
 import (
 	"encoding/json"
@@ -154,7 +154,7 @@ func findConfigsToDownload(currentApi api.Api, client rest.DynatraceClient) ([]a
 	return client.List(currentApi)
 }
 
-func downloadConfigsOfApi(theApi api.Api, values []api.Value, client rest.DynatraceClient, projectId string) []config.Config {
+func downloadConfigsOfApi(theApi api.Api, values []api.Value, client rest.DynatraceClient, projectName string) []config.Config {
 	configs := make([]config.Config, 0, len(values))
 	configsMutex := sync.Mutex{}
 
@@ -166,7 +166,7 @@ func downloadConfigsOfApi(theApi api.Api, values []api.Value, client rest.Dynatr
 		value := value
 
 		go func() {
-			conf, skipConfig := downloadConfig(theApi, value, client, projectId)
+			conf, skipConfig := downloadConfig(theApi, value, client, projectName)
 
 			if !skipConfig {
 				configsMutex.Lock()
@@ -183,8 +183,8 @@ func downloadConfigsOfApi(theApi api.Api, values []api.Value, client rest.Dynatr
 	return configs
 }
 
-func downloadConfig(theApi api.Api, value api.Value, client rest.DynatraceClient, projectId string) (conf config.Config, skipConfig bool) {
-	return downloadConfigForTesting(theApi, value, client, projectId, shouldConfigBePersisted)
+func downloadConfig(theApi api.Api, value api.Value, client rest.DynatraceClient, projectName string) (conf config.Config, skipConfig bool) {
+	return downloadConfigForTesting(theApi, value, client, projectName, shouldConfigBePersisted)
 }
 
 type shouldConfigBePersistedFunc func(a api.Api, json map[string]interface{}) bool

pkg/download/downloader/downloader_test.go → pkg/download/classic/downloader_test.go

@@ -14,7 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package downloader
+package classic
 
 import (
 	"errors"

pkg/download/downloader/test_utils.go → pkg/download/classic/test_utils.go

@@ -14,7 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package downloader
+package classic
 
 import (
 	"encoding/json"

pkg/download/settings/settings.go

@@ -0,0 +1,91 @@
+/**
+ * @license
+ * Copyright 2020 Dynatrace LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package settings
+
+import (
+	config "github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/config/v2"
+	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/config/v2/coordinate"
+	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/config/v2/parameter"
+	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/config/v2/parameter/value"
+	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/config/v2/template"
+	v2 "github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/project/v2"
+	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/rest"
+	"github.com/dynatrace-oss/dynatrace-monitoring-as-code/pkg/util/log"
+)
+
+func Download(client rest.SettingsClient) v2.ConfigsPerType {
+
+	log.Debug("Fetching schemas to download")
+	schemas, err := client.ListSchemas()
+	if err != nil {
+		log.Error("Failed to fetch all known schemas. Skipping settings download. Reason: %s", err)
+		return nil
+	}
+
+	results := make(v2.ConfigsPerType)
+
+	for _, schema := range schemas {
+		log.Debug("Downloading all settings for schema %s", schema)
+		objects, err := client.ListSettings(schema.SchemaId)
+		if err != nil {
+			log.Error("Failed to fetch all settings for schema %s", schema)
+			continue
+		}
+
+		configs := convertAllObjects(objects)
+		results[schema.SchemaId] = configs
+	}
+
+	return results
+}
+
+func convertAllObjects(objects []rest.DownloadSettingsObject) []config.Config {
+	result := make([]config.Config, 0, len(objects))
+
+	for _, o := range objects {
+		result = append(result, convertObject(o))
+	}
+
+	return result
+}
+
+func convertObject(o rest.DownloadSettingsObject) config.Config {
+
+	content := string(*o.Value)
+
+	templ := template.NewDownloadTemplate(o.ObjectId, o.ObjectId, content)
+
+	return config.Config{
+		Template: templ,
+		Coordinate: coordinate.Coordinate{
+			Project:  "project",
+			Type:     o.SchemaId,
+			ConfigId: o.ObjectId, // we need a unique id -> use the objectId as it is unique
+		},
+		Type: config.Type{
+			SchemaId:      o.SchemaId,
+			SchemaVersion: o.SchemaVersion,
+		},
+		Parameters: map[string]parameter.Parameter{
+			config.NameParameter:  &value.ValueParameter{Value: o.ObjectId},
+			config.ScopeParameter: &value.ValueParameter{Value: o.Scope},
+		},
+		References: nil,
+		Skip:       false,
+	}
+
+}