diff --git a/go.mod b/go.mod index c055103..465e15c 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/datasage-io/datasage go 1.18 -require github.com/go-sql-driver/mysql v1.6.0 +require ( + github.com/go-sql-driver/mysql v1.6.0 + github.com/lib/pq v1.10.6 +) diff --git a/go.sum b/go.sum index 20c16d6..531408f 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= +github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= diff --git a/src/adaptors/mysql.go b/src/adaptors/mysql.go index 1b08547..b5507ee 100644 --- a/src/adaptors/mysql.go +++ b/src/adaptors/mysql.go @@ -3,6 +3,7 @@ package adaptors import ( "database/sql" "fmt" + "log" _ "github.com/go-sql-driver/mysql" ) @@ -13,9 +14,9 @@ type Mysql struct { Connc *sql.DB } -func NewMysqlClient(username, password, address, dbname string) (Mysql, error) { +func NewMysqlClient(username, password, host string) (Mysql, error) { //Create data source name with given information - dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s", username, password, address, dbname) + dsn := fmt.Sprintf("%s:%s@tcp(%s)/", username, password, host) //open connection with database connc, err := sql.Open("mysql", dsn) @@ -27,3 +28,96 @@ func NewMysqlClient(username, password, address, dbname string) (Mysql, error) { func (my *Mysql) Close() { my.Connc.Close() } + +func (my Mysql) Scan() (DbMetaInfo, error) { + var dbMetaInfo DbMetaInfo + + schemas, err := my.GetSchemaDetails() + if err != nil { + return DbMetaInfo{}, err + } + + schemaData := []Schema{} + for _, sc := range schemas { + tables, err := my.GetTableNamesFromDB(sc) + if err != nil { + log.Println(err.Error()) + } + tbData := []Table{} + for _, tb := range tables { + tbcol, err := my.GetTableDetails(sc, tb) + if err != nil { + log.Println(err.Error()) + } + tbData = append(tbData, tbcol) + } + schemaData = append(schemaData, Schema{Name: sc, Tables: tbData}) + } + + dbMetaInfo.Schemas = schemaData + return dbMetaInfo, nil +} + +func (mysql Mysql) GetSchemaDetails() ([]string, error) { + + var schemaNames []string + rows, Qerr := mysql.Connc.Query("select SCHEMA_NAME from information_schema.schemata order by schema_name") + if Qerr != nil { + return schemaNames, Qerr + } + for rows.Next() { + var tmp string + if err := rows.Scan(&tmp); err != nil { + Qerr = err + } else { + schemaNames = append(schemaNames, tmp) + } + } + return schemaNames, Qerr +} + +func (mysql Mysql) GetTableNamesFromDB(dbname string) ([]string, error) { + + var tableNames []string + rows, Qerr := mysql.Connc.Query(`SELECT TABLE_NAME from information_schema.tables + WHERE table_type = 'BASE TABLE' and table_schema = ?`, dbname) + if Qerr != nil { + return tableNames, Qerr + } + for rows.Next() { + var tmp string + if err := rows.Scan(&tmp); err != nil { + Qerr = err + } else { + tableNames = append(tableNames, tmp) + } + } + return tableNames, Qerr +} + +func (mysql Mysql) GetTableDetails(dbname string, tableName string) (Table, error) { + var tableInfo Table + + tableInfo.Name = tableName + + rows, Qerr := mysql.Connc.Query(`SELECT COLUMN_NAME as column_name, + COLUMN_TYPE as column_type, + COLUMN_COMMENT as column_comment + from information_schema.columns + where table_schema = ? and table_name = ?`, dbname, tableName) + if Qerr != nil { + return tableInfo, Qerr + } + + for rows.Next() { + tempCol := Column{} + if err := rows.Scan( + &tempCol.ColumnName, &tempCol.ColumnType, + &tempCol.ColumnComment); err != nil { + Qerr = err + } else { + tableInfo.Cols = append(tableInfo.Cols, tempCol) + } + } + return tableInfo, Qerr +} diff --git a/src/adaptors/postgres.go b/src/adaptors/postgres.go index ed360ca..c94f535 100644 --- a/src/adaptors/postgres.go +++ b/src/adaptors/postgres.go @@ -1 +1,128 @@ package adaptors + +import ( + "database/sql" + "fmt" + "log" + + _ "github.com/lib/pq" +) + +type Postgres struct { + ConecStr string + Connc *sql.DB +} + +//BUG: solve Postgres client connection string error +func NewPostgresClient(username, password, host string) (Postgres, error) { + + dsn := fmt.Sprintf("host=%s port=5432 dbname='postgres' user=%s password=%s sslmode=disable", + host, username, password) + + connc, err := sql.Open("postgres", dsn) + + return Postgres{ConecStr: dsn, Connc: connc}, err +} + +func (pg Postgres) Close() error { + return pg.Connc.Close() +} +func (pg Postgres) Scan() (DbMetaInfo, error) { + var dbMetaInfo DbMetaInfo + + schemas, err := pg.GetSchemaDetails() + if err != nil { + return DbMetaInfo{}, err + } + + schemaData := []Schema{} + for _, sc := range schemas { + tables, err := pg.GetTableNamesFromDB(sc) + if err != nil { + log.Println(err.Error()) + } + tbData := []Table{} + for _, tb := range tables { + tbcol, err := pg.GetTableDetails(sc, tb) + if err != nil { + log.Println(err.Error()) + } + tbData = append(tbData, tbcol) + } + schemaData = append(schemaData, Schema{Name: sc, Tables: tbData}) + } + + dbMetaInfo.Schemas = schemaData + return dbMetaInfo, nil + +} +func (pg Postgres) GetSchemaDetails() ([]string, error) { + + var schemaNames []string + rows, Qerr := pg.Connc.Query("SELECT datname FROM pg_database") + if Qerr != nil { + return schemaNames, Qerr + } + for rows.Next() { + var tmp string + if err := rows.Scan(&tmp); err != nil { + Qerr = err + } else { + schemaNames = append(schemaNames, tmp) + } + } + return schemaNames, Qerr +} + +func (pg Postgres) GetTableNamesFromDB(dbname string) ([]string, error) { + + var tableNames []string + rows, Qerr := pg.Connc.Query(`SELECT table_name FROM information_schema.tables + WHERE table_schema='public' AND table_type = 'BASE TABLE' AND table_catalog=$1`, dbname) + if Qerr != nil { + return tableNames, Qerr + } + for rows.Next() { + var tmp string + if err := rows.Scan(&tmp); err != nil { + Qerr = err + } else { + tableNames = append(tableNames, tmp) + } + } + return tableNames, Qerr +} + +func (pg Postgres) GetTableDetails(dbname string, tableName string) (Table, error) { + var tableInfo Table + + tableInfo.Name = tableName + + q := fmt.Sprintf(` + SELECT column_name, + data_type as column_type, + col_description('public.%s'::regclass, ordinal_position) + from information_schema.columns where table_name=$1 + `, tableName) + + rows, Qerr := pg.Connc.Query(q, tableName) + if Qerr != nil { + return tableInfo, Qerr + } + + for rows.Next() { + tempCol := Column{} + tempComment := sql.NullString{} + if err := rows.Scan( + &tempCol.ColumnName, &tempCol.ColumnType, + &tempComment); err != nil { + Qerr = err + } else { + if tempComment.Valid { + tempCol.ColumnComment = tempComment.String + } + tableInfo.Cols = append(tableInfo.Cols, tempCol) + } + } + return tableInfo, Qerr +} diff --git a/src/application/main.go b/src/application/main.go deleted file mode 100644 index 6336804..0000000 --- a/src/application/main.go +++ /dev/null @@ -1,3 +0,0 @@ -package application - -func main() {} diff --git a/src/main.go b/src/main.go new file mode 100644 index 0000000..a7596b6 --- /dev/null +++ b/src/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "log" + + "github.com/datasage-io/datasage/src/adaptors" +) + +func main() { + adpt, err := adaptors.New(adaptors.AdaptorConfig{ + Type: "mysql", + Username: "appuser", + Password: "appuserpassword", + Host: "localhost:3306"}) + + if err != nil { + log.Fatal(err.Error()) + } + info, err := adpt.Scan() + + if err != nil { + log.Fatal(err.Error()) + } + + for _, sc := range info.Schemas { + log.Println("DB name:= ", sc.Name) + + for _, tb := range sc.Tables { + log.Println("Table Name:= ", tb.Name) + + for _, cols := range tb.Cols { + log.Println("columns:= ", cols) + } + } + } +}