Merge branch 'main' into go-tidy

datasage-io · Jul 20, 2022 · 78148cd · 78148cd
2 parents 820f169 + 4e3a56f
commit 78148cd
Show file tree

Hide file tree

Showing 7 changed files with 235 additions and 124 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -26,8 +26,9 @@ jobs:
           distribution: goreleaser
           version: latest
           args: release --rm-dist
-          workdir: datasage/src
           goarch:
             - amd64
+          workdir: src
+          binary: datasage
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/src/classifiers/classfication_handler.go b/src/classifiers/classfication_handler.go
@@ -44,126 +44,139 @@ func Run() {
 
 		datasources, err := st.GetDataSources()
 		if err != nil {
-			fmt.Println("ListDatasources error  ")
+			fmt.Println("Datasources not found  ")
 		}
 		for _, datasource := range datasources {
 			fmt.Println(datasource)
+			ScanDataSource(datasource)
 
-			adaptor, err := adaptors.New(adaptors.AdaptorConfig{
-				Type:     datasource.Dstype,
-				Username: datasource.User,
-				Password: datasource.Password,
-				Host:     datasource.Host,
-			})
+			log.Info().Msgf("scan completed for datasource: %v", datasource.Host)
+		}
+	}
 
-			if err != nil {
-				log.Error().Err(err).Msg("Internal Error")
-				continue
-			}
+}
 
-			//get all classes
-			classes, err := st.GetClasses()
-			if err != nil {
-				log.Error().Err(err).Msg("Internal Error")
+func ScanDataSource(datasource storage.DpDataSource) error {
 
-			}
-			log.Info().Msgf("no of classes:=%v ", len(classes))
+	fmt.Println("starting  scan for ", datasource)
+	st, err := storage.GetStorageInstance()
+	if err != nil {
+		log.Error().Err(err).Msg("Internal Error")
+		return err
+	}
+	adaptor, err := adaptors.New(adaptors.AdaptorConfig{
+		Type:     datasource.Dstype,
+		Username: datasource.User,
+		Password: datasource.Password,
+		Host:     datasource.Host,
+	})
+
+	if err != nil {
+		log.Error().Err(err).Msg("Internal Error")
+		return err
+	}
+
+	//get all classes
+	classes, err := st.GetClasses()
+	if err != nil {
+		log.Error().Err(err).Msg("Internal Error")
+
+	}
+	log.Info().Msgf("no of classes:=%v ", len(classes))
 
-			//get all tags
-			tags, err := st.GetTags()
-			if err != nil {
-				log.Error().Err(err).Msg("Internal Error")
+	//get all tags
+	tags, err := st.GetTags()
+	if err != nil {
+		log.Error().Err(err).Msg("Internal Error")
+	}
+	log.Info().Msgf("no of tags:=%v ", len(tags))
+	columsCount := 0
+	tableCount := 0
+	dBCount := 0
+	skipDBs := []string{"mysql", "performance_schema", "datadefender"}
+	info, err := adaptor.Scan()
+	for _, sc := range info.Schemas {
+		dBCount = dBCount + 1
+		log.Info().Msgf("DB name:= %v", sc.Name)
+		skip := false
+		for _, skipDB := range skipDBs {
+
+			if skipDB == sc.Name {
+				log.Info().Msgf("skip DB name:= ", sc.Name)
+				skip = true
 			}
-			log.Info().Msgf("no of tags:=%v ", len(tags))
-			columsCount := 0
-			tableCount := 0
-			dBCount := 0
-			skipDBs := []string{"mysql", "performance_schema", "datadefender"}
-			info, err := adaptor.Scan()
-			for _, sc := range info.Schemas {
-				dBCount = dBCount + 1
-				log.Info().Msgf("DB name:= %v", sc.Name)
-				skip := false
-				for _, skipDB := range skipDBs {
-
-					if skipDB == sc.Name {
-						log.Info().Msgf("skip DB name:= ", sc.Name)
-						skip = true
-					}
-				}
-				if skip == true {
-					continue
-				}
+		}
+		if skip == true {
+			continue
+		}
 
-				dpDbTables := []storage.DpDbTable{}
-				for _, tb := range sc.Tables {
-					tableCount = tableCount + 1
-					//dpDbColumn := storage.DpDbColumn
-
-					dpDbColumns := []storage.DpDbColumn{}
-
-					for _, cols := range tb.Cols {
-						columsCount = columsCount + 1
-						colName, err := removeSpecialChars(cols.ColumnName)
-						if err != nil {
-							log.Error().Err(err).Msg("Internal Error")
-							continue
-						}
-						relatedtags, _ := st.GetAssociatedTags(colName)
-						relatedclasses, _ := st.GetAssociatedClasses(colName)
-
-						//if err != nil {
-						//	log.Println(err.Error())
-						//}
-						tags := []string{}
-						classes := []string{}
-						if len(relatedclasses) > 0 {
-							for _, relatedclass := range relatedclasses {
-								log.Info().Msgf("Class:= %v", relatedclass.Class)
-								classes = append(classes, relatedclass.Class)
-							}
-						}
-						if len(relatedtags) > 0 {
-							for _, relatedtag := range relatedtags {
-								log.Info().Msgf("TagName:= %v", relatedtag.TagName)
-								//tags = tags + ";" + relatedtag.TagName
-								tags = append(tags, relatedtag.TagName)
-							}
-						}
-						col := storage.DpDbColumn{
-							ColumnName:    colName,
-							ColumnType:    cols.ColumnType,
-							ColumnComment: cols.ColumnComment,
-							Tags:          strings.Join(tags, ";"),
-							Classes:       strings.Join(classes, ";"),
-						}
-						dpDbColumns = append(dpDbColumns, col)
-					}
-					dpDbTable := storage.DpDbTable{
+		dpDbTables := []storage.DpDbTable{}
+		for _, tb := range sc.Tables {
+			tableCount = tableCount + 1
+			//dpDbColumn := storage.DpDbColumn
 
-						Name:        tb.Name,
-						Tags:        "",
-						DpDbColumns: dpDbColumns,
-					}
-					dpDbTables = append(dpDbTables, dpDbTable)
+			dpDbColumns := []storage.DpDbColumn{}
 
+			for _, cols := range tb.Cols {
+				columsCount = columsCount + 1
+				colName, err := removeSpecialChars(cols.ColumnName)
+				if err != nil {
+					log.Error().Err(err).Msg("Internal Error")
+					continue
 				}
-				schema := storage.DpDbDatabase{
-					DsKey:      datasource.DsKey,
-					Name:       sc.Name,
-					Type:       datasource.Dstype,
-					DpDbTables: dpDbTables,
+				relatedtags, _ := st.GetAssociatedTags(colName)
+				relatedclasses, _ := st.GetAssociatedClasses(colName)
+
+				//if err != nil {
+				//	log.Println(err.Error())
+				//}
+				tags := []string{}
+				classes := []string{}
+				if len(relatedclasses) > 0 {
+					for _, relatedclass := range relatedclasses {
+						log.Info().Msgf("Class:= %v", relatedclass.Class)
+						classes = append(classes, relatedclass.Class)
+					}
 				}
-				err := st.SetSchemaData(schema)
-				if err != nil {
-					fmt.Println(err)
+				if len(relatedtags) > 0 {
+					for _, relatedtag := range relatedtags {
+						log.Info().Msgf("TagName:= %v", relatedtag.TagName)
+						//tags = tags + ";" + relatedtag.TagName
+						tags = append(tags, relatedtag.TagName)
+					}
+				}
+				col := storage.DpDbColumn{
+					ColumnName:    colName,
+					ColumnType:    cols.ColumnType,
+					ColumnComment: cols.ColumnComment,
+					Tags:          strings.Join(tags, ";"),
+					Classes:       strings.Join(classes, ";"),
 				}
+				dpDbColumns = append(dpDbColumns, col)
+			}
+			dpDbTable := storage.DpDbTable{
 
+				Name:        tb.Name,
+				Tags:        "",
+				DpDbColumns: dpDbColumns,
 			}
-			log.Info().Msgf("scan completed for datasource: %v", datasource.Host)
+			dpDbTables = append(dpDbTables, dpDbTable)
+
+		}
+		schema := storage.DpDbDatabase{
+			DsKey:      datasource.DsKey,
+			Name:       sc.Name,
+			Type:       datasource.Dstype,
+			DpDbTables: dpDbTables,
+		}
+		err := st.SetSchemaData(schema)
+		if err != nil {
+			fmt.Println(err)
 		}
-	}
 
+	}
+	log.Info().Msgf("scan completed for datasource: %v", datasource.Host)
+	return nil
 }
 
 //removeSpecialChars - Removes the special characters from the string.

diff --git a/src/integrations/http_server.go b/src/integrations/http_server.go
@@ -32,6 +32,7 @@ func loghandler(w http.ResponseWriter, r *http.Request) {
 
 }
 
+
 var config, _ = ReadLogConfig("/etc/datasage/conf/datasage.yaml")
 
 func RunServer() {

diff --git a/src/main.go b/src/main.go
@@ -18,7 +18,7 @@ var configFilePath *string
 var log *zerolog.Logger = logger.GetInstance()
 
 func main() {
-	configFilePath = flag.String("config-path", "conf/", "conf/")
+	configFilePath = flag.String("config-path", "/etc/datasage/conf/", "conf/")
 	loadConfig()
 	wg.Add(1)
 	//start integration component dependent servers

diff --git a/src/server/grpcServer.go b/src/server/grpcServer.go
@@ -9,6 +9,7 @@ import (
 
 	logger "github.com/datasage-io/datasage/src/logger"
 
+	"github.com/datasage-io/datasage/src/classifiers"
 	classpb "github.com/datasage-io/datasage/src/proto/class"
 	ds "github.com/datasage-io/datasage/src/proto/datasource"
 	tagpb "github.com/datasage-io/datasage/src/proto/tag"
@@ -43,7 +44,24 @@ type ClassServer struct {
 
 func (d *ClassServer) AddClass(ctx context.Context, in *classpb.CreateRequest) (*classpb.MessageResponse, error) {
 	fmt.Println("AddClass : ", in)
-	return nil, nil
+	st, err := storage.GetStorageInstance()
+	if err != nil {
+		log.Error().Err(err).Msg("Internal Error")
+	} else {
+		rules := in.GetTag()
+		log.Debug().Msgf("rules %v", rules)
+		for _, rule := range rules {
+			log.Debug().Msgf("Rule  className %v", rule)
+
+			err := st.AddClass(in.GetDescription(), rule, in.GetName())
+			if err != nil {
+				log.Error().Err(err).Msg("Internal Error")
+			}
+		}
+		return &classpb.MessageResponse{Message: "Class Added sucessfully"}, nil
+	}
+	return &classpb.MessageResponse{Message: "Error in adding Class "}, nil
+
 }
 func (d *ClassServer) ListClass(ctx context.Context, in *classpb.ListRequest) (*classpb.ListResponse, error) {
 	fmt.Println("ListClass : ", in)
@@ -57,7 +75,6 @@ func (d *ClassServer) ListClass(ctx context.Context, in *classpb.ListRequest) (*
 	classesOut := []*classpb.ClassResponse{}
 	for _, class := range classes {
 		log.Debug().Msgf("ListTag %v", class)
-		//classes := []string{tag.Rule}
 		classOut := &classpb.ClassResponse{
 			Id:          strconv.Itoa(class.Id),
 			Name:        class.Rule,
@@ -157,11 +174,16 @@ func (d *DatasourceServer) AddDatasource(ctx context.Context, in *ds.AddRequest)
 		DsKey:        uuid.New().String(),
 	}
 
-	err1 := st.AddDataSource(storageDpDataSourceObj)
-	if err1 != nil {
+	err = st.AddDataSource(storageDpDataSourceObj)
+	if err != nil {
 		return &ds.MessageResponse{Message: "Error"}, nil
+	} else {
+		err := classifiers.ScanDataSource(storageDpDataSourceObj)
+		if err != nil {
+			return &ds.MessageResponse{Message: "Failed to Scan Datasource "}, nil
+		}
 	}
-	return &ds.MessageResponse{Message: "Sucess"}, nil
+	return &ds.MessageResponse{Message: "Data Source added for Scaning"}, nil
 }
 func (d *DatasourceServer) ListDatasource(ctx context.Context, in *ds.ListRequest) (*ds.ListResponse, error) {
 	fmt.Println("List Datasource Request ", in)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -32,6 +32,7 @@ func loghandler(w http.ResponseWriter, r *http.Request) {

		}


		var config, _ = ReadLogConfig("/etc/datasage/conf/datasage.yaml")

		func RunServer() {
Expand Down