From 218c059782a4b880fdf3e5a17d980920b6906291 Mon Sep 17 00:00:00 2001 From: illia-li Date: Sat, 17 Jun 2023 22:58:37 -0400 Subject: [PATCH] *added ColumnName in IndexDef *added function that`s adding referenses in 'IndexDef' on 'ColumnDef' for loaded schema from file *some fixes --- cmd/gemini/root.go | 3 +- cmd/gemini/schema.json | 120 ++++++++++++++++++++------ pkg/generators/column.go | 25 +++++- pkg/generators/statement_generator.go | 2 +- pkg/jobs/gen_check_stmt.go | 2 +- pkg/jobs/gen_utils_test.go | 16 ++-- pkg/jobs/jobs.go | 2 +- pkg/typedef/columns.go | 28 ++++-- pkg/typedef/columns_test.go | 54 ++++++++++-- pkg/typedef/table.go | 2 +- pkg/typedef/typedef.go | 5 +- 11 files changed, 206 insertions(+), 53 deletions(-) diff --git a/cmd/gemini/root.go b/cmd/gemini/root.go index 80c8093a..af795477 100644 --- a/cmd/gemini/root.go +++ b/cmd/gemini/root.go @@ -125,8 +125,9 @@ func readSchema(confFile string) (*typedef.Schema, error) { schemaBuilder := builders.NewSchemaBuilder() schemaBuilder.Keyspace(shm.Keyspace) - for _, tbl := range shm.Tables { + for t, tbl := range shm.Tables { schemaBuilder.Table(tbl) + generators.AddReferencesForIndexes(shm.Tables[t]) } return schemaBuilder.Build(), nil } diff --git a/cmd/gemini/schema.json b/cmd/gemini/schema.json index aba2bc8f..591f2d7d 100644 --- a/cmd/gemini/schema.json +++ b/cmd/gemini/schema.json @@ -1,6 +1,14 @@ { "keyspace": { - "name": "ks1" + "name": "ks1", + "replication": { + "class": "SimpleStrategy", + "replication_factor": 1 + }, + "oracle_replication": { + "class": "SimpleStrategy", + "replication_factor": 1 + } }, "tables": [ { @@ -42,18 +50,6 @@ }, { "name": "col1", - "type": "timestamp" - }, - { - "name": "col2", - "type": "decimal" - }, - { - "name": "col3", - "type": "uuid" - }, - { - "name": "col4", "type": { "complex_type": "map", "key_type": "boolean", @@ -62,7 +58,7 @@ } }, { - "name": "col5", + "name": "col2", "type": { "complex_type": "tuple", "value_types": [ @@ -73,7 +69,7 @@ } }, { - "name": "col6", + "name": "col3", "type": { "complex_type": "list", "value_type": "int", @@ -81,30 +77,106 @@ } }, { - "name": "col7", + "name": "col4", "type": { "complex_type": "set", "value_type": "int", "frozen": true } + }, + { + "name": "col5", + "type": "ascii" + }, + { + "name": "col6", + "type": "bigint" + }, + { + "name": "col7", + "type": "blob" + }, + { + "name": "col8", + "type": "boolean" + }, + { + "name": "col9", + "type": "date" + }, + { + "name": "col10", + "type": "decimal" + }, + { + "name": "col11", + "type": "double" + }, + { + "name": "col12", + "type": "duration" + }, + { + "name": "col13", + "type": "float" + }, + { + "name": "col14", + "type": "inet" + }, + { + "name": "col15", + "type": "int" + }, + { + "name": "col16", + "type": "smallint" + }, + { + "name": "col17", + "type": "text" + }, + { + "name": "col19", + "type": "timestamp" + }, + { + "name": "col20", + "type": "timeuuid" + }, + { + "name": "col21", + "type": "tinyint" + }, + { + "name": "col22", + "type": "uuid" + }, + { + "name": "col23", + "type": "varchar" + }, + { + "name": "col24", + "type": "varint" } ], "indexes": [ { - "name": "col0_idx", - "column": "col0" + "index_name": "col0_idx", + "column_name": "col56" }, { - "name": "col1_idx", - "column": "col1" + "index_name": "col1_idx", + "column_name": "col6" }, { - "name": "col2_idx", - "column": "col2" + "index_name": "col2_idx", + "column_name": "col7" }, { - "name": "col3_idx", - "column": "col3" + "index_name": "col3_idx", + "column_name": "col8" } ], "known_issues": { diff --git a/pkg/generators/column.go b/pkg/generators/column.go index 7e5fdd4a..ccc437f6 100644 --- a/pkg/generators/column.go +++ b/pkg/generators/column.go @@ -15,6 +15,8 @@ package generators import ( + "fmt" + "github.com/scylladb/gemini/pkg/typedef" ) @@ -23,7 +25,11 @@ func CreateIndexesForColumn(table *typedef.Table, maxIndexes int) []typedef.Inde indexes := make([]typedef.IndexDef, 0, maxIndexes) for i, col := range table.Columns { if col.Type.Indexable() && typedef.TypesForIndex.Contains(col.Type) { - indexes = append(indexes, typedef.IndexDef{Name: GenIndexName(table.Name+"_col", i), Column: table.Columns[i]}) + indexes = append(indexes, typedef.IndexDef{ + IndexName: GenIndexName(table.Name+"_col", i), + ColumnName: table.Columns[i].Name, + Column: table.Columns[i], + }) createdCount++ } if createdCount == maxIndexes { @@ -32,3 +38,20 @@ func CreateIndexesForColumn(table *typedef.Table, maxIndexes int) []typedef.Inde } return indexes } + +func AddReferencesForIndexes(table *typedef.Table) { + wrongIndex := -1 + for i, index := range table.Indexes { + for c, column := range table.Columns { + if index.ColumnName == column.Name { + table.Indexes[i].Column = table.Columns[c] + break + } else if len(table.Columns) == c { + wrongIndex = i + } + } + } + if wrongIndex != -1 { + panic(fmt.Sprintf("wrong column_name in index defenition:%+v", table.Indexes[wrongIndex])) + } +} diff --git a/pkg/generators/statement_generator.go b/pkg/generators/statement_generator.go index 4bda53e3..30e358af 100644 --- a/pkg/generators/statement_generator.go +++ b/pkg/generators/statement_generator.go @@ -106,7 +106,7 @@ func GetCreateSchema(s *typedef.Schema) []string { createTable := GetCreateTable(t, s.Keyspace) stmts = append(stmts, createTable) for _, idef := range t.Indexes { - stmts = append(stmts, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s.%s (%s)", idef.Name, s.Keyspace.Name, t.Name, idef.Column.Name)) + stmts = append(stmts, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s.%s (%s)", idef.IndexName, s.Keyspace.Name, t.Name, idef.ColumnName)) } for _, mv := range t.MaterializedViews { var ( diff --git a/pkg/jobs/gen_check_stmt.go b/pkg/jobs/gen_check_stmt.go index 41565ced..9a5d9dfb 100644 --- a/pkg/jobs/gen_check_stmt.go +++ b/pkg/jobs/gen_check_stmt.go @@ -521,7 +521,7 @@ func genSingleIndexQuery( builder := qb.Select(s.Keyspace.Name + "." + t.Name) builder.AllowFiltering() for i := 0; i < idxCount; i++ { - builder = builder.Where(qb.Eq(t.Indexes[i].Column.Name)) + builder = builder.Where(qb.Eq(t.Indexes[i].ColumnName)) values = append(values, t.Indexes[i].Column.Type.GenValue(r, p)...) typs = append(typs, t.Indexes[i].Column.Type) } diff --git a/pkg/jobs/gen_utils_test.go b/pkg/jobs/gen_utils_test.go index c3b014db..c68fd802 100644 --- a/pkg/jobs/gen_utils_test.go +++ b/pkg/jobs/gen_utils_test.go @@ -476,15 +476,19 @@ func createIdxFromColumns(t testInterface, table *typedef.Table, all bool) (inde switch all { case true: for i := range table.Columns { - var index typedef.IndexDef - index.Name = table.Columns[i].Name + "_idx" - index.Column = table.Columns[i] + index := typedef.IndexDef{ + IndexName: table.Columns[i].Name + "_idx", + ColumnName: table.Columns[i].Name, + Column: table.Columns[i], + } indexes = append(indexes, index) } default: - var index typedef.IndexDef - index.Name = table.Columns[0].Name + "_idx" - index.Column = table.Columns[0] + index := typedef.IndexDef{ + IndexName: table.Columns[0].Name + "_idx", + ColumnName: table.Columns[0].Name, + Column: table.Columns[0], + } indexes = append(indexes, index) } diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index cf503a93..1ba7f148 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -325,7 +325,7 @@ func ddl( defer table.Unlock() ddlStmts, err := GenDDLStmt(schema, table, r, p, sc) if err != nil { - logger.Error("Failed! Mutation statement generation failed", zap.Error(err)) + logger.Error("Failed! DDl Mutation statement generation failed", zap.Error(err)) globalStatus.WriteErrors.Add(1) return err } diff --git a/pkg/typedef/columns.go b/pkg/typedef/columns.go index 06b42a34..dbf267e6 100644 --- a/pkg/typedef/columns.go +++ b/pkg/typedef/columns.go @@ -16,11 +16,9 @@ package typedef import ( "encoding/json" - "fmt" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "golang.org/x/exp/rand" ) @@ -29,7 +27,7 @@ type ColumnDef struct { Name string `json:"name"` } -var ErrorWrongColTypeDefinition = errors.New("wrong column type definition") +var ErrSchemaValidation = errors.New("validation failed") func (cd *ColumnDef) IsValidForPrimaryKey() bool { for _, pkType := range PkTypes { @@ -49,15 +47,15 @@ func (cd *ColumnDef) UnmarshalJSON(data []byte) error { if err != nil { typeMap, typeOk := dataMap["type"] if !typeOk { - return errors.Wrap(ErrorWrongColTypeDefinition, fmt.Sprintf("input: %s\n", dataMap)) + return errors.Wrapf(ErrSchemaValidation, "missing definition of column 'type': [%T]%+[1]v", dataMap) } complexTypeMap, typeMapOk := typeMap.(map[string]interface{}) if !typeMapOk { - return errors.Wrap(ErrorWrongColTypeDefinition, fmt.Sprintf("input: %s\n", typeMap)) + return errors.Wrapf(ErrSchemaValidation, "unknown definition column 'type': [%T]%+[1]v", typeMap) } complexType, complexTypeOk := complexTypeMap["complex_type"] if !complexTypeOk { - return errors.Wrap(ErrorWrongColTypeDefinition, fmt.Sprintf("input: %s\n", complexTypeMap)) + return errors.Wrapf(ErrSchemaValidation, "missing definition of column 'complex_type': [%T]%+[1]v", complexTypeMap) } switch complexType { case TYPE_LIST, TYPE_SET: @@ -69,7 +67,7 @@ func (cd *ColumnDef) UnmarshalJSON(data []byte) error { case TYPE_UDT: t, err = GetUDTTypeColumn(dataMap) default: - return errors.Wrap(ErrorWrongColTypeDefinition, fmt.Sprintf("input: %s\n", complexType)) + return errors.Wrapf(ErrSchemaValidation, "unknown 'complex_type': [%T]%+[1]v", complexType) } if err != nil { return err @@ -305,6 +303,22 @@ func GetSimpleTypeColumn(data map[string]interface{}) (*ColumnDef, error) { if err != nil { return nil, err } + if st.Name == "" { + return nil, errors.Wrapf(ErrSchemaValidation, "wrong definition of column 'name' [%T]%+[1]v", data) + } + if st.Type == "" { + return nil, errors.Wrapf(ErrSchemaValidation, "empty definition of column 'type' [%T]%+[1]v", data) + } + + knownType := false + for _, sType := range AllTypes { + if sType == st.Type { + knownType = true + } + } + if !knownType { + return nil, errors.Wrapf(ErrSchemaValidation, "not simple type in column 'type' [%T]%+[1]v", data) + } return &ColumnDef{ Name: st.Name, Type: st.Type, diff --git a/pkg/typedef/columns_test.go b/pkg/typedef/columns_test.go index 3a46a868..439960f8 100644 --- a/pkg/typedef/columns_test.go +++ b/pkg/typedef/columns_test.go @@ -17,6 +17,7 @@ package typedef_test import ( "encoding/json" "fmt" + "os" "strings" "testing" @@ -41,7 +42,8 @@ var allSimpleTypes = []typedef.SimpleType{ typedef.TYPE_INT, typedef.TYPE_SMALLINT, typedef.TYPE_TEXT, - typedef.TYPE_TIME, + // TODO: Add support for time when gocql bug is fixed. + // typedef.TYPE_TIME, typedef.TYPE_TIMESTAMP, typedef.TYPE_TIMEUUID, typedef.TYPE_TINYINT, @@ -79,7 +81,7 @@ func TestColumnMarshalUnmarshal(t *testing.T) { Name: "udt1", }, //nolint:lll - expected: "{\"type\":{\"complex_type\":\"udt\",\"value_types\":{\"col_ascii\":\"ascii\",\"col_bigint\":\"bigint\",\"col_blob\":\"blob\",\"col_boolean\":\"boolean\",\"col_date\":\"date\",\"col_decimal\":\"decimal\",\"col_double\":\"double\",\"col_duration\":\"duration\",\"col_float\":\"float\",\"col_inet\":\"inet\",\"col_int\":\"int\",\"col_smallint\":\"smallint\",\"col_text\":\"text\",\"col_time\":\"time\",\"col_timestamp\":\"timestamp\",\"col_timeuuid\":\"timeuuid\",\"col_tinyint\":\"tinyint\",\"col_uuid\":\"uuid\",\"col_varchar\":\"varchar\",\"col_varint\":\"varint\"},\"type_name\":\"udt1\",\"frozen\":false},\"name\":\"udt1\"}", + expected: "{\"type\":{\"complex_type\":\"udt\",\"value_types\":{\"col_ascii\":\"ascii\",\"col_bigint\":\"bigint\",\"col_blob\":\"blob\",\"col_boolean\":\"boolean\",\"col_date\":\"date\",\"col_decimal\":\"decimal\",\"col_double\":\"double\",\"col_duration\":\"duration\",\"col_float\":\"float\",\"col_inet\":\"inet\",\"col_int\":\"int\",\"col_smallint\":\"smallint\",\"col_text\":\"text\",\"col_timestamp\":\"timestamp\",\"col_timeuuid\":\"timeuuid\",\"col_tinyint\":\"tinyint\",\"col_uuid\":\"uuid\",\"col_varchar\":\"varchar\",\"col_varint\":\"varint\"},\"type_name\":\"udt1\",\"frozen\":false},\"name\":\"udt1\"}", }) for id := range testCases { @@ -98,8 +100,6 @@ func TestColumnMarshalUnmarshal(t *testing.T) { var unmarshaledDef typedef.ColumnDef err = json.Unmarshal(marshaledData, &unmarshaledDef) if err != nil { - fmt.Printf("_______%s", marshaledData) - fmt.Printf("_______%s", unmarshaledDef) t.Fatal(err.Error()) } @@ -133,6 +133,42 @@ func TestMarshalUnmarshal(t *testing.T) { } } +func TestMarshalUnmarshalSchemaExample(t *testing.T) { + filePath := "cmd/gemini/schema.json" + dir, _ := os.Getwd() + dir, _, _ = strings.Cut(dir, "pkg") + filePath = dir + filePath + + testSchemaLoaded, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("failed to open schema example json file %s, error:%s", filePath, err) + } + + var testSchema typedef.Schema + err = json.Unmarshal(testSchemaLoaded, &testSchema) + if err != nil { + t.Fatalf("failed to unmarshal schema example from json file %s, error:%s", filePath, err) + } + + opts := cmp.Options{ + cmp.AllowUnexported(typedef.Table{}, typedef.MaterializedView{}), + cmpopts.IgnoreUnexported(typedef.Table{}, typedef.MaterializedView{}), + } + + testSchemaMarshaled, err := json.MarshalIndent(testSchema, " ", " ") + if err != nil { + t.Fatalf("unable to marshal schema example json, error=%s\n", err) + } + testSchemaUnMarshaled := typedef.Schema{} + if err = json.Unmarshal(testSchemaMarshaled, &testSchemaUnMarshaled); err != nil { + t.Fatalf("unable to unmarshal json, error=%s\n", err) + } + + if diff := cmp.Diff(testSchema, testSchemaUnMarshaled, opts); diff != "" { + t.Errorf("schema not the same after marshal/unmarshal, diff=%s", diff) + } +} + func TestPrimitives(t *testing.T) { sc := &typedef.SchemaConfig{ MaxPartitionKeys: 3, @@ -293,12 +329,14 @@ func getTestSchema() *typedef.Schema { } sch.Tables[0].Indexes = []typedef.IndexDef{ { - Name: generators.GenIndexName(sch.Tables[0].Name+"_col", 0), - Column: columns[0], + IndexName: generators.GenIndexName(sch.Tables[0].Name+"_col", 0), + ColumnName: columns[0].Name, + Column: columns[0], }, { - Name: generators.GenIndexName(sch.Tables[0].Name+"_col", 1), - Column: columns[1], + IndexName: generators.GenIndexName(sch.Tables[0].Name+"_col", 1), + ColumnName: columns[1].Name, + Column: columns[1], }, } diff --git a/pkg/typedef/table.go b/pkg/typedef/table.go index 48bf8129..0ea3445f 100644 --- a/pkg/typedef/table.go +++ b/pkg/typedef/table.go @@ -98,7 +98,7 @@ func (t *Table) ValidColumnsForDelete() Columns { if len(t.Indexes) != 0 { for _, idx := range t.Indexes { for j := range validCols { - if validCols[j].Name == idx.Column.Name { + if validCols[j].Name == idx.ColumnName { validCols = append(validCols[:j], validCols[j+1:]...) break } diff --git a/pkg/typedef/typedef.go b/pkg/typedef/typedef.go index 6ecf5c8d..f559d0d8 100644 --- a/pkg/typedef/typedef.go +++ b/pkg/typedef/typedef.go @@ -34,8 +34,9 @@ type ( } IndexDef struct { - Column *ColumnDef - Name string `json:"name"` + Column *ColumnDef + IndexName string `json:"index_name"` + ColumnName string `json:"column_name"` } PartitionRangeConfig struct {