Add support for creating space during dashboard restore (#1543)

* Create spaces for dashboard

Signed-off-by: Md. Ishtiaq Islam <[email protected]>

* Update deps

Signed-off-by: Md. Ishtiaq Islam <[email protected]>

* Fix error log

Signed-off-by: Md. Ishtiaq Islam <[email protected]>

---------

Signed-off-by: Md. Ishtiaq Islam <[email protected]>

go.mod

@@ -7,6 +7,7 @@ toolchain go1.22.2
 require (
 	github.com/spf13/cobra v1.8.0
 	go.bytebuilders.dev/license-verifier/kubernetes v0.14.0
+	golang.org/x/text v0.14.0
 	gomodules.xyz/flags v0.1.3
 	gomodules.xyz/go-sh v0.1.0
 	gomodules.xyz/logs v0.0.7
@@ -18,8 +19,8 @@ require (
 	kmodules.xyz/client-go v0.29.13
 	kmodules.xyz/custom-resources v0.29.1
 	kmodules.xyz/offshoot-api v0.29.0
-	kubedb.dev/apimachinery v0.41.0
-	kubedb.dev/db-client-go v0.0.11-0.20240208083800-50462091d436
+	kubedb.dev/apimachinery v0.44.0
+	kubedb.dev/db-client-go v0.0.15-0.20240419064300-f4fab3d22391
 	sigs.k8s.io/controller-runtime v0.17.2
 	stash.appscode.dev/apimachinery v0.34.0
 )
@@ -29,6 +30,7 @@ require (
 	github.com/PuerkitoBio/purell v1.2.1 // indirect
 	github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cert-manager/cert-manager v1.13.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
@@ -81,7 +83,6 @@ require (
 	golang.org/x/oauth2 v0.15.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/term v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	gomodules.xyz/clock v0.0.0-20200817085942-06523dba733f // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
@@ -104,6 +105,7 @@ require (
 	kmodules.xyz/monitoring-agent-api v0.29.0 // indirect
 	kmodules.xyz/objectstore-api v0.29.1 // indirect
 	kmodules.xyz/prober v0.29.0 // indirect
+	kubeops.dev/petset v0.0.5 // indirect
 	sigs.k8s.io/gateway-api v0.8.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect

go.sum

@@ -31,6 +31,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/cert-manager/cert-manager v1.13.3 h1:3R4G0RI7K0OkTZhWlVOC5SGZMYa2NwqmQJoyKydrz/M=
 github.com/cert-manager/cert-manager v1.13.3/go.mod h1:BM2+Pt/NmSv1Zr25/MHv6BgIEF9IUxA1xAjp80qkxgc=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -332,8 +334,8 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
-go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -587,10 +589,12 @@ kmodules.xyz/offshoot-api v0.29.0 h1:GHLhxxT9jU1N8+FvOCCeJNyU5g0duYS46UGrs6AHNLY
 kmodules.xyz/offshoot-api v0.29.0/go.mod h1:5NxhBblXoDHWStx9HCDJR2KFTwYjEZ7i1Id3jelIunw=
 kmodules.xyz/prober v0.29.0 h1:Ex7m4F9rH7uWNNJlLgP63ROOM+nUATJkC2L5OQ7nwMg=
 kmodules.xyz/prober v0.29.0/go.mod h1:UtK+HKyI1lFLEKX+HFLyOCVju6TO93zv3kwGpzqmKOo=
-kubedb.dev/apimachinery v0.41.0 h1:VbGQnH3YL7ICFvnCjAumlnL3HeZzg5F4F+flpjOSnG4=
-kubedb.dev/apimachinery v0.41.0/go.mod h1:rNWsbBzdnZA8G2FE8igi+nsGnlWqYurC+i3RFFDwluc=
-kubedb.dev/db-client-go v0.0.11-0.20240208083800-50462091d436 h1:JmSnrw+IdzbkgisjDQ5X0oo5jSts4OhBLHgEw6SL3qs=
-kubedb.dev/db-client-go v0.0.11-0.20240208083800-50462091d436/go.mod h1:sadvZUXo7tWz/75gdW7wpqeR25bi9kI/h2djyYLjN0A=
+kubedb.dev/apimachinery v0.44.0 h1:R7tiR35tmjJxNlpHUS8PVPmSrnFMfLycDE0c9XEU1/A=
+kubedb.dev/apimachinery v0.44.0/go.mod h1:7daaaWragCFLV38plrrJtsOuzinBSX3enMpliqlm3Uo=
+kubedb.dev/db-client-go v0.0.15-0.20240419064300-f4fab3d22391 h1:QqKQl3YVwOXcmHsCo5k6DSCW7qeUFJlhn7/XyJ4DmuA=
+kubedb.dev/db-client-go v0.0.15-0.20240419064300-f4fab3d22391/go.mod h1:Vv6x2vfdOGzjWa7iOOuMYyABXXAIkJpJ5eZ79GcgNqc=
+kubeops.dev/petset v0.0.5 h1:VVXi39JhjondlbHyZ98z0MLp6VCmiCMinL59K48Y2zA=
+kubeops.dev/petset v0.0.5/go.mod h1:ijtKT1HlAht2vBEZj5LW7C00XEs3B0d1VdCQgd5V4cA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 sigs.k8s.io/gateway-api v0.8.0 h1:isQQ3Jx2qFP7vaA3ls0846F0Amp9Eq14P08xbSwVbQg=
 sigs.k8s.io/gateway-api v0.8.0/go.mod h1:okOnjPNBFbIS/Rw9kAhuIUaIkLhTKEu+ARIuXk2dgaM=

pkg/backup.go

@@ -18,6 +18,7 @@ package pkg
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -313,8 +314,17 @@ func (opt *esOptions) dumpDashboardObjects(appBinding *appcatalog.AppBinding) er
 		return err
 	}
 
+	data, err := json.Marshal(spaces)
+	if err != nil {
+		return fmt.Errorf("failed to marshal spaces: %w", err)
+	}
+
+	if err = os.WriteFile(filepath.Join(opt.interimDataDir, SpacesInfoFile), data, os.ModePerm); err != nil {
+		return fmt.Errorf("failed to write spaces info: %w", err)
+	}
+
 	for _, space := range spaces {
-		response, err := dashboardClient.ExportSavedObjects(space)
+		response, err := dashboardClient.ExportSavedObjects(space.Id)
 		if err != nil {
 			return err
 		}
@@ -328,7 +338,7 @@ func (opt *esOptions) dumpDashboardObjects(appBinding *appcatalog.AppBinding) er
 			return fmt.Errorf("failed to export dashboard saved objects %s", string(body))
 		}
 
-		if err = os.WriteFile(opt.getDashboardFilePath(space), body, os.ModePerm); err != nil {
+		if err = os.WriteFile(opt.getDashboardFilePath(space.Id), body, os.ModePerm); err != nil {
 			return err
 		}
 	}

pkg/restore.go

@@ -18,18 +18,22 @@ package pkg
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"stash.appscode.dev/apimachinery/apis"
 	api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
 	"stash.appscode.dev/apimachinery/pkg/restic"
 
 	"github.com/spf13/cobra"
 	license "go.bytebuilders.dev/license-verifier/kubernetes"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 	"gomodules.xyz/flags"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -38,6 +42,7 @@ import (
 	appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
 	appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
 	v1 "kmodules.xyz/offshoot-api/api/v1"
+	"kubedb.dev/db-client-go/elasticsearchdashboard"
 )
 
 func NewCmdRestore() *cobra.Command {
@@ -257,13 +262,24 @@ func (opt *esOptions) restoreDashboardObjects(appBinding *appcatalog.AppBinding)
 		return err
 	}
 
-	spaces, err := dashboardClient.ListSpaces()
+	spaces, err := opt.getSpaces()
+	if err != nil {
+		return err
+	}
+
+	existingSpaces, err := dashboardClient.ListSpaces()
 	if err != nil {
 		return err
 	}
 
 	for _, space := range spaces {
-		response, err := dashboardClient.ImportSavedObjects(space, opt.getDashboardFilePath(space))
+		if !isExist(existingSpaces, space.Id) {
+			if err = dashboardClient.CreateSpace(space); err != nil {
+				return fmt.Errorf("failed to create space %s: %w", space.Id, err)
+			}
+		}
+
+		response, err := dashboardClient.ImportSavedObjects(space.Id, opt.getDashboardFilePath(space.Id))
 		if err != nil {
 			return err
 		}
@@ -278,10 +294,65 @@ func (opt *esOptions) restoreDashboardObjects(appBinding *appcatalog.AppBinding)
 		}
 
 		// delete the dashboard file(s) as it is not required for restoring the dumps
-		if err = clearFile(opt.getDashboardFilePath(space)); err != nil {
+		if err = clearFile(opt.getDashboardFilePath(space.Id)); err != nil {
 			return err
 		}
 	}
 
 	return nil
 }
+
+func (opt *esOptions) getSpaces() ([]elasticsearchdashboard.Space, error) {
+	if _, err := os.Stat(filepath.Join(opt.interimDataDir, SpacesInfoFile)); os.IsNotExist(err) {
+		spaces := make([]elasticsearchdashboard.Space, 0)
+		if err = filepath.Walk(opt.interimDataDir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if filepath.Ext(info.Name()) == DashboardSavedObjectsExt {
+				id, _ := strings.CutSuffix(info.Name(), DashboardSavedObjectsExt)
+				spaces = append(spaces, elasticsearchdashboard.Space{
+					Id:   id,
+					Name: cases.Title(language.English).String(strings.Replace(id, "-", " ", -1)),
+				})
+			}
+
+			return nil
+		}); err != nil {
+			return nil, err
+		}
+
+		if len(spaces) == 0 {
+			return nil, fmt.Errorf("no spaces found in interim data directory")
+		}
+
+		return spaces, nil
+	} else {
+		data, err := os.ReadFile(filepath.Join(opt.interimDataDir, SpacesInfoFile))
+		if err != nil {
+			return nil, fmt.Errorf("failed to read spaces info %w", err)
+		}
+
+		var spaces []elasticsearchdashboard.Space
+		if err = json.Unmarshal(data, &spaces); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal spaces info %w", err)
+		}
+
+		// delete the spaces info file as it is not required for restoring the dumps
+		if err = clearFile(filepath.Join(opt.interimDataDir, SpacesInfoFile)); err != nil {
+			return nil, err
+		}
+
+		return spaces, nil
+	}
+}
+
+func isExist(existingSpaces []elasticsearchdashboard.Space, spaceId string) bool {
+	for _, space := range existingSpaces {
+		if space.Id == spaceId {
+			return true
+		}
+	}
+	return false
+}

pkg/utils.go

@@ -50,11 +50,13 @@ import (
 )
 
 const (
-	ESUser              = "ADMIN_USERNAME"
-	ESPassword          = "ADMIN_PASSWORD"
-	MultiElasticDumpCMD = "multielasticdump"
-	ESCACertFile        = "root.pem"
-	ESAuthFile          = "auth.txt"
+	ESUser                   = "ADMIN_USERNAME"
+	ESPassword               = "ADMIN_PASSWORD"
+	MultiElasticDumpCMD      = "multielasticdump"
+	ESCACertFile             = "root.pem"
+	ESAuthFile               = "auth.txt"
+	SpacesInfoFile           = "spaces.json"
+	DashboardSavedObjectsExt = ".ndjson"
 )
 
 type esOptions struct {
@@ -294,5 +296,5 @@ func (opt esOptions) getDashboardClient(appBinding *appcatalog.AppBinding) (*es_
 }
 
 func (opt esOptions) getDashboardFilePath(space string) string {
-	return filepath.Join(opt.interimDataDir, space+".ndjson")
+	return filepath.Join(opt.interimDataDir, space+DashboardSavedObjectsExt)
 }