Adds support for Org based override of Watched Folders (#212)

* Adds support for Org based override of Watched Folders

Fixes #193

ChangeLog: 
* Added a cleanup after test functionality.
* BUG: Fixing CLI Parser small bug.
* Adding CLI test for devel srvinfo
* Updating go version to 1.21.3 to address secrutiy vulnerability
* Addressing code review comments

.github/workflows/code_scanners.yml

@@ -29,7 +29,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.21.2"
+          go-version: "1.21.3"
           cache: false
       - name: Install Task
         run: sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d

.github/workflows/go.yml

@@ -11,7 +11,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go: [ {version: 1.21.0, token: 1}, {version: 1.21.0, token: 0}]
+        go: [ {version: 1.21.3, token: 1}, {version: 1.21.3, token: 0}]
         grafana: [ 10.1.4 ]
 
     env:
@@ -40,11 +40,11 @@ jobs:
         run: | 
           echo "token IS $TEST_TOKEN_CONFIG"
       - name: Calc coverage
-        if: "${{ matrix.go.version == '1.21.0' && matrix.grafana == '10.1.4' && matrix.go.token == '0' }}"
+        if: "${{ matrix.go.version == '1.21.3' && matrix.grafana == '10.1.4' && matrix.go.token == '0' }}"
         run: |
           go test -v -covermode=atomic -coverprofile=coverage.out ./...
       - name: Convert coverage.out to coverage.lcov
-        if: "${{ matrix.go.version == '1.21.0' && matrix.grafana == '10.1.4' && matrix.go.token == '0' }}"
+        if: "${{ matrix.go.version == '1.21.3' && matrix.grafana == '10.1.4' && matrix.go.token == '0' }}"
         uses: jandelgado/[email protected]
       - name: Test 
         if: "${{ matrix.go.token == '1' }}"

Taskfile.yml

@@ -107,14 +107,16 @@ tasks:
   test:
       desc: "test check"
       cmds:
-        - go test -v ./... -cover
+        - go test -v -coverpkg=./... -covermode=atomic -coverprofile=coverage.out  ./... 
+        - go tool cover -html=coverage.out
       env:
         GRAFANA_INTEGRATION: "1"
         TEST_TOKEN_CONFIG: "0"
   test_tokens:
       desc: "test Token Based Only"
       cmds:
-        - go test -v ./... -cover
+        - go test -v -coverpkg=./... -covermode=atomic -coverprofile=coverage.out ./...
+        - go tool cover -html=coverage.out
       env:
         GRAFANA_INTEGRATION: "1"
         TEST_TOKEN_CONFIG: "1"

cmd/backup/conections_test.go → cmd/test/conections_test.go

@@ -1,9 +1,8 @@
-package backup_test
+package test
 
 import (
 	"github.com/esnet/gdg/cmd"
 	"github.com/esnet/gdg/cmd/support"
-	"github.com/esnet/gdg/cmd/test_tools"
 	"github.com/esnet/gdg/internal/service"
 	"github.com/esnet/gdg/internal/service/mocks"
 	"github.com/esnet/grafana-swagger-api-golang/goclient/models"
@@ -38,7 +37,7 @@ func TestConnectionCommand(t *testing.T) {
 			response.GrafanaSvc = getMockSvc
 		}
 	}
-	r, w, cleanup := test_tools.InterceptStdout()
+	r, w, cleanup := InterceptStdout()
 	data, err := os.ReadFile("../../config/testing.yml")
 	assert.Nil(t, err)
 

cmd/test/devel_test.go

@@ -0,0 +1,50 @@
+package test
+
+import (
+	"github.com/esnet/gdg/cmd"
+	"github.com/esnet/gdg/cmd/support"
+	"github.com/esnet/gdg/internal/service/mocks"
+	"github.com/stretchr/testify/assert"
+	"strings"
+	"testing"
+)
+
+func TestDevelSrvInfo(t *testing.T) {
+	var execMe = func(mock *mocks.GrafanaService, data []byte, optionMockSvc func() support.RootOption) error {
+		expected := make(map[string]interface{})
+		expected["Database"] = "db"
+		expected["Commit"] = "commit"
+		expected["Version"] = "version"
+
+		mock.On("GetServerInfo").Return(expected)
+		err := cmd.Execute(string(data), []string{"tools", "devel", "srvinfo"}, optionMockSvc())
+		return err
+	}
+	outStr, closeReader := setupAndExecuteMockingServices(t, execMe)
+	defer closeReader()
+
+	assert.True(t, strings.Contains(outStr, "Version:"))
+	assert.True(t, strings.Contains(outStr, "Database:"))
+	assert.True(t, strings.Contains(outStr, "Commit:"))
+}
+
+func TestDevelSrvCompletion(t *testing.T) {
+	fn := func(args []string) func(mock *mocks.GrafanaService, data []byte, optionMockSvc func() support.RootOption) error {
+		return func(mock *mocks.GrafanaService, data []byte, optionMockSvc func() support.RootOption) error {
+			err := cmd.Execute(string(data), args, optionMockSvc())
+			return err
+		}
+	}
+
+	outStr, closeReader := setupAndExecuteMockingServices(t, fn([]string{"tools", "devel", "completion", "fish"}))
+	assert.True(t, strings.Contains(outStr, "fish"))
+	assert.True(t, strings.Contains(outStr, "__completion_prepare_completions"))
+	closeReader()
+	outStr, closeReader = setupAndExecuteMockingServices(t, fn([]string{"tools", "devel", "completion", "bash"}))
+	assert.True(t, strings.Contains(outStr, "bash"))
+	assert.True(t, strings.Contains(outStr, "flag_parsing_disabled"))
+	closeReader()
+	outStr, closeReader = setupAndExecuteMockingServices(t, fn([]string{"tools", "devel", "completion", "zsh"}))
+	assert.True(t, strings.Contains(outStr, "shellCompDirectiveKeepOrder"))
+	closeReader()
+}

cmd/test/support.go

@@ -0,0 +1,66 @@
+package test
+
+import (
+	"github.com/esnet/gdg/cmd/support"
+	"github.com/esnet/gdg/internal/service"
+	"github.com/esnet/gdg/internal/service/mocks"
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"io"
+	"os"
+	"testing"
+)
+
+// setupAndExecuteMockingServices  will create a mock for varous required entities allowing to test the CLI flag parsing
+// process: function that setups mocks and invokes the Execute command
+func setupAndExecuteMockingServices(t *testing.T, process func(mock *mocks.GrafanaService, data []byte, optionMockSvc func() support.RootOption) error) (string, func()) {
+	testSvc := new(mocks.GrafanaService)
+	getMockSvc := func() service.GrafanaService {
+		return testSvc
+	}
+
+	optionMockSvc := func() support.RootOption {
+		return func(response *support.RootCommand) {
+			response.GrafanaSvc = getMockSvc
+		}
+	}
+
+	r, w, cleanup := InterceptStdout()
+	data, err := os.ReadFile("../../config/testing.yml")
+	assert.Nil(t, err)
+
+	err = process(testSvc, data, optionMockSvc)
+	assert.Nil(t, err)
+	defer cleanup()
+	err = w.Close()
+	if err != nil {
+		log.Warn("unable to close write stream")
+	}
+	clean := func() {
+		defer r.Close()
+	}
+	out, _ := io.ReadAll(r)
+	outStr := string(out)
+	return outStr, clean
+
+}
+
+// InterceptStdout is a test helper function that will redirect all stdout in and out to a different file stream.
+// It returns the stdout, stderr, and a function to be invoked to close the streams.
+func InterceptStdout() (*os.File, *os.File, func()) {
+	backupStd := os.Stdout
+	backupErr := os.Stderr
+	r, w, _ := os.Pipe()
+	//Restore streams
+	log.SetOutput(w)
+	cleanup := func() {
+		os.Stdout = backupStd
+		os.Stderr = backupErr
+		log.SetOutput(os.Stdout)
+	}
+	os.Stdout = w
+	os.Stderr = w
+
+	return r, w, cleanup
+
+}

cmd/version_test.go → cmd/test/version_test.go

@@ -1,42 +1,26 @@
-package cmd
+package test
 
 import (
 	"fmt"
+	"github.com/esnet/gdg/cmd"
 	"github.com/esnet/gdg/cmd/support"
-	"github.com/esnet/gdg/cmd/test_tools"
 	"github.com/esnet/gdg/internal/service"
 	"github.com/esnet/gdg/internal/service/mocks"
 	"github.com/esnet/gdg/internal/version"
 	"github.com/stretchr/testify/assert"
-	"io"
 	"os"
 	"strings"
 	"testing"
 )
 
 func TestVersionCommand(t *testing.T) {
-	testSvc := new(mocks.GrafanaService)
-	getMockSvc := func() service.GrafanaService {
-		return testSvc
+	var execMe = func(mock *mocks.GrafanaService, data []byte, optionMockSvc func() support.RootOption) error {
+		err := cmd.Execute(string(data), []string{"version"}, optionMockSvc())
+		return err
 	}
+	outStr, closeReader := setupAndExecuteMockingServices(t, execMe)
+	defer closeReader()
 
-	optionMockSvc := func() support.RootOption {
-		return func(response *support.RootCommand) {
-			response.GrafanaSvc = getMockSvc
-		}
-	}
-	path, _ := os.Getwd()
-	fmt.Println(path)
-	r, w, cleanup := test_tools.InterceptStdout()
-	data, err := os.ReadFile("../config/testing.yml")
-	assert.Nil(t, err)
-
-	err = Execute(string(data), []string{"version"}, optionMockSvc())
-	assert.Nil(t, err)
-	defer cleanup()
-	w.Close()
-	out, _ := io.ReadAll(r)
-	outStr := string(out)
 	assert.True(t, strings.Contains(outStr, "Build Date:"))
 	assert.True(t, strings.Contains(outStr, "Git Commit:"))
 	assert.True(t, strings.Contains(outStr, "Version:"))
@@ -59,10 +43,10 @@ func TestVersionErrCommand(t *testing.T) {
 	}
 	path, _ := os.Getwd()
 	fmt.Println(path)
-	data, err := os.ReadFile("../config/testing.yml")
+	data, err := os.ReadFile("../../config/testing.yml")
 	assert.Nil(t, err)
 
-	err = Execute(string(data), []string{"dumb", "dumb"}, optionMockSvc())
+	err = cmd.Execute(string(data), []string{"dumb", "dumb"}, optionMockSvc())
 	assert.NotNil(t, err)
 	assert.Equal(t, err.Error(), `command error: unknown command "dumb" for "gdg"`)
 }

cmd/tools/auth_service_accounts.go

@@ -42,7 +42,7 @@ func newListServiceAccountCmd() simplecobra.Commander {
 		NameP:        "list",
 		Short:        description,
 		Long:         description,
-		CommandsList: []simplecobra.Commander{newTokensCmd()},
+		CommandsList: []simplecobra.Commander{},
 		RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error {
 			rootCmd.TableObj.AppendHeader(table.Row{"id", "service name", "role", "tokens", "token id", "token name", "expiration"})
 			apiKeys := rootCmd.GrafanaSvc().ListServiceAccounts()
@@ -83,7 +83,7 @@ func newDeleteServiceAccountTokensCmd() simplecobra.Commander {
 		NameP:        "clearTokens",
 		Short:        description,
 		Long:         description,
-		CommandsList: []simplecobra.Commander{newTokensCmd()},
+		CommandsList: []simplecobra.Commander{},
 		RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error {
 			if len(args) < 1 {
 				return errors.New("requires a service account ID to be specified")
@@ -116,7 +116,7 @@ func newDeleteServiceAccountCmd() simplecobra.Commander {
 		NameP:        "clear",
 		Short:        description,
 		Long:         description,
-		CommandsList: []simplecobra.Commander{newTokensCmd()},
+		CommandsList: []simplecobra.Commander{},
 		RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error {
 			savedFiles := rootCmd.GrafanaSvc().DeleteAllServiceAccounts()
 			log.Infof("Delete Service Accounts for context: '%s'", config.Config().AppConfig.GetContext())
@@ -140,7 +140,7 @@ func newServiceAccount() simplecobra.Commander {
 		NameP:        "newService",
 		Short:        description,
 		Long:         description,
-		CommandsList: []simplecobra.Commander{newTokensCmd()},
+		CommandsList: []simplecobra.Commander{},
 		RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error {
 			if len(args) < 2 {
 				return errors.New("requires a key name and a role('admin','viewer','editor') [ttl optional] ")

config/importer-example.yml

@@ -5,7 +5,7 @@ storage_engine:
     kind: cloud
     cloud_type: s3
     bucket_name: ""
-## The configuration below is mainly intended for OSS alternatives like ceph and minio.  If you use a known cloud provider
+  ## The configuration below is mainly intended for OSS alternatives like ceph and minio.  If you use a known cloud provider
   ## like aws, gcs, azure please setup the auth using the provided tooling from the cloud provider.
   # For example, having a valid AWS bucket configured in ~/.aws/credentials will be sufficient without needing to provide the auth in the config.
   ### valid boolean values can be represented as true, "true", or "1"
@@ -18,7 +18,7 @@ storage_engine:
     access_id: ""  ## this value can also be read from: AWS_ACCESS_KEY. config file is given precedence
     secret_key: ""  ## same as above, can be read from: AWS_SECRET_KEY with config file is given precedence.
     init_bucket: "true" ## Only supported for custom workflows. Will attempt to create a bucket if one does not exist.
-    endpoint: "http://localhost:9000"  
+    endpoint: "http://localhost:9000"
     ssl_enabled: "false"
 
 contexts:
@@ -115,6 +115,11 @@ contexts:
     watched:
       - Folder1
       - Folder2
+    watched_folders_override:
+      - organization_id: 0
+        folders:
+          - General
+          - SpecialFolder
 
 global:
   debug: true

config/testing.yml

@@ -83,6 +83,7 @@ contexts:
       - Folder1
       - Folder2
 
+
 global:
   debug: true
   ignore_ssl_errors: false ##when set to true will ignore invalid SSL errors

docker/Dockerfile

@@ -1,5 +1,5 @@
 # Build Stage
-FROM golang:1.21.2  AS build-stage
+FROM golang:1.21.3  AS build-stage
 
 LABEL app="build-gdg"
 LABEL REPO="https://github.com/esnet/gdg"
@@ -15,7 +15,7 @@ WORKDIR /go/src/github.com/esnet/gdg
 RUN make build-alpine
 
 # Final Stage
-FROM golang:1.21.2 
+FROM golang:1.21.3 
 
 ARG GIT_COMMIT
 ARG VERSION

go.mod

@@ -1,6 +1,6 @@
 module github.com/esnet/gdg
 
-go 1.21.2
+go 1.21.3
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
@@ -147,7 +147,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/mod v0.13.0 // indirect
-	golang.org/x/net v0.16.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect

go.sum

@@ -664,6 +664,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
 golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=