Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FEAT] Database schema in the Gateway #578

Merged
merged 13 commits into from
Dec 3, 2024
Merged
Prev Previous commit
Next Next commit
test(connections): add test for all new helper functions
rogefm committed Dec 3, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 83c4326bbffe7b94dc5529c3b0fd9859d8c3b777
8 changes: 7 additions & 1 deletion gateway/api/connections/connections.go
Original file line number Diff line number Diff line change
@@ -533,7 +533,13 @@ printjson(result);`
func GetDatabaseSchemas(c *gin.Context) {
ctx := storagev2.ParseContext(c)
connNameOrID := c.Param("nameOrID")
dbName := c.Query("database")
dbName := c.Query("database") // Criar uma regex para validar o nome do banco de dados e remover sql injectionn 422

// Validate database name to prevent SQL injection
if err := validateDatabaseName(dbName); err != nil {
c.JSON(http.StatusUnprocessableEntity, gin.H{"message": err.Error()})
return
}

conn, err := FetchByName(ctx, connNameOrID)
if err != nil {
39 changes: 39 additions & 0 deletions gateway/api/connections/helpers.go
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import (
"regexp"
"slices"
"strings"
"unicode"

"github.com/gin-gonic/gin"
rogefm marked this conversation as resolved.
Show resolved Hide resolved
pb "github.com/hoophq/hoop/common/proto"
@@ -327,6 +328,7 @@ func parseMongoDBSchema(output string) (openapi.ConnectionSchemaResponse, error)
response.Schemas = append(response.Schemas, *schema)
}

// fmt.Printf("response parseMongoDBSchema", response)
return response, nil
}

@@ -463,3 +465,40 @@ func organizeSchemaResponse(rows []map[string]interface{}) openapi.ConnectionSch

return response
}

// validateDatabaseName returns an error if the database name contains invalid characters
func validateDatabaseName(dbName string) error {
// Regular expression that allows only:
// - Letters (a-z, A-Z)
// - Numbers (0-9)
// - Underscores (_)
// - Hyphens (-)
// - Dots (.)
// With length between 1 and 128 characters
re := regexp.MustCompile(`^[a-zA-Z0-9_\-\.]{1,128}$`)

if !re.MatchString(dbName) {
return fmt.Errorf("invalid database name. Only alphanumeric characters, underscore, hyphen and dot are allowed with length between 1 and 128 characters")
}

// Some databases don't allow names starting with numbers
if unicode.IsDigit(rune(dbName[0])) {
return fmt.Errorf("database name cannot start with a number")
}

// Check common reserved words
reservedWords := []string{
"master", "tempdb", "model", "msdb", // SQL Server
"postgres", "template0", "template1", // PostgreSQL
"mysql", "information_schema", "performance_schema", // MySQL
}

dbNameLower := strings.ToLower(dbName)
for _, word := range reservedWords {
if dbNameLower == word {
return fmt.Errorf("database name cannot be a reserved word: %s", word)
}
}

return nil
}
616 changes: 375 additions & 241 deletions gateway/api/connections/helpers_test.go
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ import (
"net/http"
"net/url"
"reflect"
"strings"
"testing"

pb "github.com/hoophq/hoop/common/proto"
@@ -434,268 +435,401 @@ func TestParseMongoDBSchemaWithComplexIndexes(t *testing.T) {
}
}

// Helper functions
func findTable(tables []openapi.ConnectionTable, name string) *openapi.ConnectionTable {
for i := range tables {
if tables[i].Name == name {
return &tables[i]
}
}
return nil
}

func findColumn(columns []openapi.ConnectionColumn, name string) *openapi.ConnectionColumn {
for i := range columns {
if columns[i].Name == name {
return &columns[i]
}
}
return nil
}

func TestParseSQLSchema(t *testing.T) {
testInput := `schema_name object_type object_name column_name column_type not_null column_default is_primary_key is_foreign_key index_name index_columns index_is_unique index_is_primary
public table categories category integer t nextval('categories_category_seq'::regclass) f f categories_pkey {category} t t
public table categories categoryname character varying(50) t f f categories_pkey {category} t t
public table cust_hist customerid integer t f t ix_cust_hist_customerid {customerid} f f
public table cust_hist orderid integer t f f ix_cust_hist_customerid {customerid} f f
public table customers customerid integer t nextval('customers_customerid_seq'::regclass) f f customers_pkey {customerid} t t
public table customers email character varying(50) f f f ix_cust_username {username} t f`

result, err := parseSQLSchema(testInput, pb.ConnectionTypePostgres)
assert.NoError(t, err)

// Basic structure validation
assert.Len(t, result.Schemas, 1)
schema := result.Schemas[0]
assert.Equal(t, "public", schema.Name)
assert.Len(t, schema.Tables, 3)

// Test categories table
categoriesTable := findTable(schema.Tables, "categories")
assert.NotNil(t, categoriesTable)
assert.Len(t, categoriesTable.Columns, 2)
assert.Len(t, categoriesTable.Indexes, 1)

// Test categories columns
catIdCol := findColumn(categoriesTable.Columns, "category")
assert.NotNil(t, catIdCol)
assert.Equal(t, "integer", catIdCol.Type)
assert.Equal(t, "nextval('categories_category_seq'::regclass)", catIdCol.DefaultValue)
assert.False(t, catIdCol.Nullable)

catNameCol := findColumn(categoriesTable.Columns, "categoryname")
assert.NotNil(t, catNameCol)
assert.Equal(t, "character varying(50)", catNameCol.Type)
assert.False(t, catNameCol.Nullable)
assert.Empty(t, catNameCol.DefaultValue)

// Test categories indexes
assert.Equal(t, "categories_pkey", categoriesTable.Indexes[0].Name)
assert.Equal(t, []string{"{category}"}, categoriesTable.Indexes[0].Columns)
assert.True(t, categoriesTable.Indexes[0].IsUnique)
assert.True(t, categoriesTable.Indexes[0].IsPrimary)

// Test cust_hist table
custHistTable := findTable(schema.Tables, "cust_hist")
assert.NotNil(t, custHistTable)
assert.Len(t, custHistTable.Columns, 2)

// Test cust_hist foreign key
custIdCol := findColumn(custHistTable.Columns, "customerid")
assert.NotNil(t, custIdCol)
assert.True(t, custIdCol.IsForeignKey)
assert.False(t, custIdCol.IsPrimaryKey)

// Test customers table with nullable column
customersTable := findTable(schema.Tables, "customers")
assert.NotNil(t, customersTable)
emailCol := findColumn(customersTable.Columns, "email")
assert.NotNil(t, emailCol)
assert.True(t, emailCol.Nullable)

// Test edge cases
emptyInput := ""
emptyResult, err := parseSQLSchema(emptyInput, pb.ConnectionTypePostgres)
assert.NoError(t, err)
assert.Len(t, emptyResult.Schemas, 0)

invalidInput := "invalid\tformat\tdata"
invalidResult, err := parseSQLSchema(invalidInput, pb.ConnectionTypePostgres)
assert.NoError(t, err)
assert.Len(t, invalidResult.Schemas, 0)
}

func TestOrganizeSchemaResponse(t *testing.T) {
// Test case with multiple tables, views, columns and indexes
input := []map[string]interface{}{
{
"schema_name": "public",
"object_type": "table",
"object_name": "categories",
"column_name": "category",
"column_type": "integer",
"not_null": true,
"column_default": "nextval('categories_category_seq'::regclass)",
"is_primary_key": true,
"is_foreign_key": false,
"index_name": "categories_pkey",
"index_columns": []string{"category"},
"index_is_unique": true,
"index_is_primary": true,
},
{
"schema_name": "public",
"object_type": "table",
"object_name": "categories",
"column_name": "categoryname",
"column_type": "character varying(50)",
"not_null": true,
"column_default": "",
"is_primary_key": false,
"is_foreign_key": false,
"index_name": "categories_pkey",
"index_columns": []string{"category"},
"index_is_unique": true,
"index_is_primary": true,
},
{
"schema_name": "public",
"object_type": "view",
"object_name": "active_categories",
"column_name": "category",
"column_type": "integer",
"not_null": true,
"column_default": "",
"is_primary_key": false,
"is_foreign_key": false,
"index_name": "",
"index_columns": []string{},
"index_is_unique": false,
"index_is_primary": false,
},
}

result := organizeSchemaResponse(input)

// Validate basic structure
assert.Len(t, result.Schemas, 1)
schema := result.Schemas[0]
assert.Equal(t, "public", schema.Name)
assert.Len(t, schema.Tables, 1)
assert.Len(t, schema.Views, 1)

// Validate table
table := schema.Tables[0]
assert.Equal(t, "categories", table.Name)
assert.Len(t, table.Columns, 2)
assert.Len(t, table.Indexes, 1)

// Validate columns
idCol := findColumn(table.Columns, "category")
assert.NotNil(t, idCol)
assert.Equal(t, "integer", idCol.Type)
assert.Equal(t, "nextval('categories_category_seq'::regclass)", idCol.DefaultValue)
assert.False(t, idCol.Nullable)
assert.True(t, idCol.IsPrimaryKey)

nameCol := findColumn(table.Columns, "categoryname")
assert.NotNil(t, nameCol)
assert.Equal(t, "character varying(50)", nameCol.Type)
assert.False(t, nameCol.Nullable)
assert.Empty(t, nameCol.DefaultValue)
assert.False(t, nameCol.IsPrimaryKey)

// Validate index
assert.Equal(t, "categories_pkey", table.Indexes[0].Name)
assert.Equal(t, []string{"category"}, table.Indexes[0].Columns)
assert.True(t, table.Indexes[0].IsUnique)
assert.True(t, table.Indexes[0].IsPrimary)

// Validate view
view := schema.Views[0]
assert.Equal(t, "active_categories", view.Name)
assert.Len(t, view.Columns, 1)

viewCol := view.Columns[0]
assert.Equal(t, "category", viewCol.Name)
assert.Equal(t, "integer", viewCol.Type)
assert.False(t, viewCol.Nullable)
assert.Empty(t, viewCol.DefaultValue)
assert.False(t, viewCol.IsPrimaryKey)

// Test case with multiple schemas
multiSchemaInput := []map[string]interface{}{
{
"schema_name": "public",
"object_type": "table",
"object_name": "table1",
"column_name": "id",
"column_type": "integer",
"not_null": true,
"column_default": "",
"is_primary_key": true,
"is_foreign_key": false,
"index_name": "",
"index_columns": []string{},
"index_is_unique": false,
"index_is_primary": false,
},
{
"schema_name": "app",
"object_type": "table",
"object_name": "table2",
"column_name": "id",
"column_type": "integer",
"not_null": true,
"column_default": "",
"is_primary_key": true,
"is_foreign_key": false,
"index_name": "",
"index_columns": []string{},
"index_is_unique": false,
"index_is_primary": false,
},
}

multiResult := organizeSchemaResponse(multiSchemaInput)
assert.Len(t, multiResult.Schemas, 2)
assert.ElementsMatch(t, []string{"public", "app"}, []string{
multiResult.Schemas[0].Name,
multiResult.Schemas[1].Name,
})

// Test case with empty input
emptyResult := organizeSchemaResponse([]map[string]interface{}{})
assert.Len(t, emptyResult.Schemas, 0)

// Test case with multiple columns in same table
sameTableInput := []map[string]interface{}{
{
"schema_name": "public",
"object_type": "table",
"object_name": "users",
"column_name": "id",
"column_type": "integer",
"not_null": true,
"column_default": "",
"is_primary_key": true,
"is_foreign_key": false,
"index_name": "",
"index_columns": []string{},
"index_is_unique": false,
"index_is_primary": false,
},
{
"schema_name": "public",
"object_type": "table",
"object_name": "users",
"column_name": "email",
"column_type": "varchar",
"not_null": true,
"column_default": "",
"is_primary_key": false,
"is_foreign_key": false,
"index_name": "",
"index_columns": []string{},
"index_is_unique": false,
"index_is_primary": false,
},
}

sameTableResult := organizeSchemaResponse(sameTableInput)
assert.Len(t, sameTableResult.Schemas, 1)
assert.Len(t, sameTableResult.Schemas[0].Tables, 1)
assert.Len(t, sameTableResult.Schemas[0].Tables[0].Columns, 2)
}

// Test helper function to validate the output format matches the real DB output
func TestParseSQLSchemaWithRealOutput(t *testing.T) {
input := `schema_name object_type object_name column_name column_type not_null column_default is_primary_key is_foreign_key index_name index_columns index_is_unique index_is_primary
public table categories category integer t nextval('categories_category_seq'::regclass) f f categories_pkey {category} t t
public table categories categoryname character varying(50) t f f categories_pkey {category} t t
public table customers customerid integer t nextval('customers_customerid_seq'::regclass) f f customers_pkey {customerid} t t
public table customers firstname character varying(50) t f f customers_pkey {customerid} t t
public table customers email character varying(50) f f f customers_pkey {customerid} t t`

got, err := parseSQLSchema(input, pb.ConnectionTypePostgres)
assert.NoError(t, err)

// Validate basic structure
assert.Len(t, got.Schemas, 1)
assert.Equal(t, "public", got.Schemas[0].Name)

// Validate tables
assert.Len(t, got.Schemas[0].Tables, 2)

// Check categories table
categoriesTable := got.Schemas[0].Tables[0]
assert.Equal(t, "categories", categoriesTable.Name)
assert.Len(t, categoriesTable.Columns, 2)
assert.Equal(t, "category", categoriesTable.Columns[0].Name)
assert.Equal(t, "integer", categoriesTable.Columns[0].Type)
assert.Equal(t, "categoryname", categoriesTable.Columns[1].Name)

// Check customers table
customersTable := got.Schemas[0].Tables[1]
assert.Equal(t, "customers", customersTable.Name)
assert.Len(t, customersTable.Columns, 3)
assert.Equal(t, "customerid", customersTable.Columns[0].Name)
assert.Equal(t, "firstname", customersTable.Columns[1].Name)
assert.Equal(t, "email", customersTable.Columns[2].Name)
}

func TestValidateDatabaseName(t *testing.T) {
tests := []struct {
name string
input string
connType pb.ConnectionType
want openapi.ConnectionSchemaResponse
wantErr bool
name string
dbName string
wantErr bool
}{
{
name: "postgres schema",
input: `public table users id integer 0 NULL 1 0 pk_users id 1 1
public table users name varchar 0 NULL 0 0 NULL NULL 0 0
public table users email varchar 0 NULL 0 0 idx_email email 1 0`,
connType: pb.ConnectionTypePostgres,
want: openapi.ConnectionSchemaResponse{
Schemas: []openapi.ConnectionSchema{
{
Name: "public",
Tables: []openapi.ConnectionTable{
{
Name: "users",
Columns: []openapi.ConnectionColumn{
{
Name: "id",
Type: "integer",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: true,
IsForeignKey: false,
},
{
Name: "name",
Type: "varchar",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: false,
IsForeignKey: false,
},
{
Name: "email",
Type: "varchar",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: false,
IsForeignKey: false,
},
},
Indexes: []openapi.ConnectionIndex{
{
Name: "pk_users",
Columns: []string{"id"},
IsUnique: true,
IsPrimary: true,
},
{
Name: "idx_email",
Columns: []string{"email"},
IsUnique: true,
IsPrimary: false,
},
},
},
},
},
},
},
name: "valid name",
dbName: "myapp_db",
wantErr: false,
},
{
name: "mssql schema with header",
input: `schema_name object_type object_name column_name column_type not_null column_default is_primary_key is_foreign_key index_name index_columns index_is_unique index_is_primary
----------- ----------- ----------- ----------- ----------- -------- ------------- -------------- -------------- ---------- ------------- --------------- ----------------
dbo table customers id int 0 NULL 1 0 PK_customers id 1 1
dbo table customers name nvarchar 0 NULL 0 0 NULL NULL 0 0`,
connType: pb.ConnectionTypeMSSQL,
want: openapi.ConnectionSchemaResponse{
Schemas: []openapi.ConnectionSchema{
{
Name: "dbo",
Tables: []openapi.ConnectionTable{
{
Name: "customers",
Columns: []openapi.ConnectionColumn{
{
Name: "id",
Type: "int",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: true,
IsForeignKey: false,
},
{
Name: "name",
Type: "nvarchar",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: false,
IsForeignKey: false,
},
},
Indexes: []openapi.ConnectionIndex{
{
Name: "PK_customers",
Columns: []string{"id"},
IsUnique: true,
IsPrimary: true,
},
},
},
},
},
},
},
name: "valid name with dots",
dbName: "my.database.test",
wantErr: false,
},
{
name: "mysql schema with foreign key",
input: `test table orders user_id int 0 NULL 0 1 fk_user user_id 0 0
test table orders order_id int 0 NULL 1 0 pk_orders order_id 1 1`,
connType: pb.ConnectionTypeMySQL,
want: openapi.ConnectionSchemaResponse{
Schemas: []openapi.ConnectionSchema{
{
Name: "test",
Tables: []openapi.ConnectionTable{
{
Name: "orders",
Columns: []openapi.ConnectionColumn{
{
Name: "user_id",
Type: "int",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: false,
IsForeignKey: true,
},
{
Name: "order_id",
Type: "int",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: true,
IsForeignKey: false,
},
},
Indexes: []openapi.ConnectionIndex{
{
Name: "fk_user",
Columns: []string{"user_id"},
IsUnique: false,
IsPrimary: false,
},
{
Name: "pk_orders",
Columns: []string{"order_id"},
IsUnique: true,
IsPrimary: true,
},
},
},
},
},
},
},
name: "valid name with hyphens",
dbName: "my-database-test",
wantErr: false,
},
{
name: "empty string",
dbName: "",
wantErr: true,
},
{
name: "empty input",
input: "",
connType: pb.ConnectionTypePostgres,
want: openapi.ConnectionSchemaResponse{},
name: "too long name",
dbName: strings.Repeat("a", 129),
wantErr: true,
},
{
name: "invalid format",
input: "invalid\tformat",
connType: pb.ConnectionTypePostgres,
want: openapi.ConnectionSchemaResponse{},
name: "special characters",
dbName: "my@database",
wantErr: true,
},
{
name: "only footer",
input: "(3 rows affected)",
connType: pb.ConnectionTypePostgres,
want: openapi.ConnectionSchemaResponse{},
name: "spaces",
dbName: "my database",
wantErr: true,
},
{
name: "sql injection attempt",
dbName: "db; DROP TABLE users;--",
wantErr: true,
},
{
name: "starts with number",
dbName: "1database",
wantErr: true,
},
{
name: "reserved word postgres",
dbName: "postgres",
wantErr: true,
},
{
name: "reserved word master",
dbName: "master",
wantErr: true,
},
{
name: "reserved word information_schema",
dbName: "information_schema",
wantErr: true,
},
{
name: "unicode characters",
dbName: "database💾",
wantErr: true,
},
{
name: "case insensitive reserved word",
dbName: "MASTER",
wantErr: true,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseSQLSchema(tt.input, tt.connType)
err := validateDatabaseName(tt.dbName)
if (err != nil) != tt.wantErr {
t.Errorf("parseSQLSchema() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseSQLSchema() = %v, want %v", got, tt.want)
t.Errorf("validateDatabaseName() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

// TestParseSQLSchemaCompoundIndexes tests handling of compound indexes
func TestParseSQLSchemaCompoundIndexes(t *testing.T) {
input := `public table users id integer 0 NULL 1 0 pk_users id 1 1
public table users email varchar 0 NULL 0 0 idx_email_name email,name 1 0
public table users name varchar 0 NULL 0 0 idx_email_name email,name 1 0`

want := openapi.ConnectionSchemaResponse{
Schemas: []openapi.ConnectionSchema{
{
Name: "public",
Tables: []openapi.ConnectionTable{
{
Name: "users",
Columns: []openapi.ConnectionColumn{
{
Name: "id",
Type: "integer",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: true,
IsForeignKey: false,
},
{
Name: "email",
Type: "varchar",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: false,
IsForeignKey: false,
},
{
Name: "name",
Type: "varchar",
Nullable: true,
DefaultValue: "NULL",
IsPrimaryKey: false,
IsForeignKey: false,
},
},
Indexes: []openapi.ConnectionIndex{
{
Name: "pk_users",
Columns: []string{"id"},
IsUnique: true,
IsPrimary: true,
},
{
Name: "idx_email_name",
Columns: []string{"email", "name"},
IsUnique: true,
IsPrimary: false,
},
},
},
},
},
},
}

got, err := parseSQLSchema(input, pb.ConnectionTypePostgres)
if err != nil {
t.Errorf("parseSQLSchema() error = %v", err)
return
}
if !reflect.DeepEqual(got, want) {
t.Errorf("parseSQLSchema() = %v, want %v", got, want)
}
}