Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add explicit mapping for serial types #851

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

feat: add explicit mapping for serial types #851

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
61f85b0
test: add tests for all postgresql types
thiagotnunes Jul 3, 2024
4c1c6c5
Merge branch 'master' of github.com:thiagotnunes/spanner-migration-to…
thiagotnunes Jul 4, 2024
84073b8
Merge branch 'master' of github.com:thiagotnunes/spanner-migration-to…
thiagotnunes Jul 9, 2024
4d8b5cb
Merge branch 'master' of https://github.com/GoogleCloudPlatform/spann…
thiagotnunes Jul 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 41 additions & 9 deletions sources/postgres/pgdump_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,33 +45,65 @@ func TestProcessPgDump(t *testing.T) {
ty string
expected ddl.Type
}{
{"bit", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"bigint", ddl.Type{Name: ddl.Int64}},
{"bool", ddl.Type{Name: ddl.Bool}},
{"boolean", ddl.Type{Name: ddl.Bool}},
{"box", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"bytea", ddl.Type{Name: ddl.Bytes, Len: ddl.MaxLength}},
{"char", ddl.Type{Name: ddl.String, Len: int64(1)}},
{"char(42)", ddl.Type{Name: ddl.String, Len: int64(42)}},
{"cidr", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"circle", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"date", ddl.Type{Name: ddl.Date}},
{"decimal", ddl.Type{Name: ddl.Numeric}}, // pg parser maps this to numeric.
{"double precision", ddl.Type{Name: ddl.Float64}},
{"float8", ddl.Type{Name: ddl.Float64}},
{"float4", ddl.Type{Name: ddl.Float64}},
{"float8", ddl.Type{Name: ddl.Float64}},
{"inet", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"int2", ddl.Type{Name: ddl.Int64}},
{"int4", ddl.Type{Name: ddl.Int64}},
{"int8", ddl.Type{Name: ddl.Int64}},
{"integer", ddl.Type{Name: ddl.Int64}},
{"interval", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"json", ddl.Type{Name: ddl.JSON}},
{"jsonb", ddl.Type{Name: ddl.JSON}},
{"line", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"lseg", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"macaddr", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"macaddr8", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"money", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"numeric", ddl.Type{Name: ddl.Numeric}},
{"numeric(4)", ddl.Type{Name: ddl.Numeric}},
{"numeric(6, 4)", ddl.Type{Name: ddl.Numeric}},
{"path", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"pg_lsn", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"pg_snapshot", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"point", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"polygon", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"real", ddl.Type{Name: ddl.Float64}},
{"smallint", ddl.Type{Name: ddl.Int64}},
{"serial", ddl.Type{Name: ddl.Int64}},
{"serial2", ddl.Type{Name: ddl.Int64}},
{"serial4", ddl.Type{Name: ddl.Int64}},
{"serial8", ddl.Type{Name: ddl.Int64}},
{"text", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"time", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"timestamp", ddl.Type{Name: ddl.Timestamp}},
{"timestamp without time zone", ddl.Type{Name: ddl.Timestamp}},
{"timestamp(5)", ddl.Type{Name: ddl.Timestamp}},
{"timestamptz", ddl.Type{Name: ddl.Timestamp}},
{"timestamp with time zone", ddl.Type{Name: ddl.Timestamp}},
{"timestamptz(5)", ddl.Type{Name: ddl.Timestamp}},
{"timetz", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"tsquery", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"tsvector", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"txid_snapshot", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"uuid", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"varbit", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"varchar", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
{"varchar(42)", ddl.Type{Name: ddl.String, Len: int64(42)}},
{"json", ddl.Type{Name: ddl.JSON}},
{"jsonb", ddl.Type{Name: ddl.JSON}},
{"xml", ddl.Type{Name: ddl.String, Len: ddl.MaxLength}},
}
for _, tc := range scalarTests {
conv, _ := runProcessPgDump(fmt.Sprintf("CREATE TABLE t (a %s);", tc.ty))
Expand All @@ -80,7 +112,7 @@ func TestProcessPgDump(t *testing.T) {
assert.Equal(t, nil, err)
colId, err := internal.GetColIdFromSpName(conv.SpSchema[tableId].ColDefs, "a")
assert.Equal(t, nil, err)
assert.Equal(t, conv.SpSchema[tableId].ColDefs[colId].T, tc.expected, "Scalar type: "+tc.ty)
assert.Equal(t, tc.expected, conv.SpSchema[tableId].ColDefs[colId].T, "Scalar type: "+tc.ty)
}
// Next test array types and not null.
singleColTests := []struct {
Expand Down Expand Up @@ -636,15 +668,15 @@ COPY test (id, a, b, c, d) FROM stdin;
spannerData{table: "test", cols: []string{"id", "a", "b", "c", "d"}, vals: []interface{}{int64(1), int64(88), int64(44), int64(22), big.NewRat(1112469, 2500)}}},
},
{
name: "Data conversion: serial, text, timestamp, timestamptz, varchar, json",
name: "Data conversion: serial, text, timestamp, timestamptz, varchar, json, serial2, serial4, serial8",
input: `
CREATE TABLE test (id integer PRIMARY KEY, a serial, b text, c timestamp, d timestamptz, e varchar, f json, g jsonb);
COPY test (id, a, b, c, d, e, f, g) FROM stdin;
1 2 my text 2019-10-29 05:30:00 2019-10-29 05:30:00+10:30 my varchar {"k":"k1", "v":"v1"} {"k":"k2", "v":"v2"}
CREATE TABLE test (id integer PRIMARY KEY, a serial, b text, c timestamp, d timestamptz, e varchar, f json, g jsonb, h serial2, i serial4, j serial8);
COPY test (id, a, b, c, d, e, f, g, h, i, j) FROM stdin;
1 2 my text 2019-10-29 05:30:00 2019-10-29 05:30:00+10:30 my varchar {"k":"k1", "v":"v1"} {"k":"k2", "v":"v2"} 3 4 5
\.
`,
expectedData: []spannerData{
spannerData{table: "test", cols: []string{"id", "a", "b", "c", "d", "e", "f", "g"}, vals: []interface{}{int64(1), int64(2), "my text", getTime(t, "2019-10-29T05:30:00Z"), getTime(t, "2019-10-29T05:30:00+10:30"), "my varchar", "{\"k\":\"k1\", \"v\":\"v1\"}", "{\"k\":\"k2\", \"v\":\"v2\"}"}}},
spannerData{table: "test", cols: []string{"id", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, vals: []interface{}{int64(1), int64(2), "my text", getTime(t, "2019-10-29T05:30:00Z"), getTime(t, "2019-10-29T05:30:00+10:30"), "my varchar", "{\"k\":\"k1\", \"v\":\"v1\"}", "{\"k\":\"k2\", \"v\":\"v2\"}", int64(3), int64(4), int64(5)}}},
},
}
for _, tc := range multiColTests {
Expand Down
2 changes: 1 addition & 1 deletion sources/postgres/toddl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ func toSpannerTypeInternal(srcType schema.Type, spType string) (ddl.Type, []inte
// if this numeric won't fit in Spanner's NUMERIC.
return ddl.Type{Name: ddl.Numeric}, nil
}
case "serial":
case "serial", "serial2", "serial4", "serial8":
switch spType {
case ddl.String:
return ddl.Type{Name: ddl.String, Len: ddl.MaxLength}, []internal.SchemaIssue{internal.Widened, internal.Serial}
Expand Down
2 changes: 1 addition & 1 deletion webv2/api/schema.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1431,7 +1431,7 @@ func initializeTypeMap() {
}
// Initialize postgresTypeMap.
toddl = postgres.InfoSchemaImpl{}.GetToDdl()
for _, srcTypeName := range []string{"bool", "boolean", "bigserial", "bpchar", "character", "bytea", "date", "float8", "double precision", "float4", "real", "int8", "bigint", "int4", "integer", "int2", "smallint", "numeric", "serial", "text", "timestamptz", "timestamp with time zone", "timestamp", "timestamp without time zone", "varchar", "character varying", "path"} {
for _, srcTypeName := range []string{"bool", "boolean", "bigserial", "bpchar", "character", "bytea", "date", "float8", "double precision", "float4", "real", "int8", "bigint", "int4", "integer", "int2", "smallint", "numeric", "serial", "serial2", "serial4", "serial8", "text", "timestamptz", "timestamp with time zone", "timestamp", "timestamp without time zone", "varchar", "character varying", "path"} {
var l []types.TypeIssue
srcType := schema.MakeType()
srcType.Name = srcTypeName
Expand Down
Loading