From 2c0e883397afe4edb5fbf5240b473e1d3a4c8cc7 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Tue, 19 Feb 2019 17:43:23 +0800 Subject: [PATCH 01/20] update parser to latest version --- dm/ctl/common/util.go | 3 ++- go.mod | 15 +++++------ go.sum | 58 +++++++++++++++++++++++------------------- loader/convert_data.go | 3 ++- pkg/ddl/common.go | 35 +++++++++++++++++++++++++ relay/util.go | 3 ++- syncer/ddl.go | 3 ++- 7 files changed, 81 insertions(+), 39 deletions(-) create mode 100644 pkg/ddl/common.go diff --git a/dm/ctl/common/util.go b/dm/ctl/common/util.go index 82c710dfdb..ce25d9ad76 100644 --- a/dm/ctl/common/util.go +++ b/dm/ctl/common/util.go @@ -22,6 +22,7 @@ import ( "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" "github.com/pingcap/dm/dm/pb" + "github.com/pingcap/dm/pkg/ddl" "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -165,7 +166,7 @@ func ExtractSQLsFromArgs(args []string) ([]string, error) { concat := strings.TrimSpace(strings.Join(args, " ")) parser := parser.New() - nodes, err := parser.Parse(concat, "", "") + nodes, err := ddl.Parse(parser, concat, "", "") if err != nil { return nil, errors.Annotatef(err, "invalid sql '%s'", concat) } diff --git a/go.mod b/go.mod index 51d3d3e4bb..f733f3f3c9 100644 --- a/go.mod +++ b/go.mod @@ -9,16 +9,14 @@ require ( github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/go-sql-driver/mysql v1.4.1 - github.com/gogo/protobuf v1.1.1 + github.com/gogo/protobuf v1.2.0 github.com/golang/protobuf v1.2.0 github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 + github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 github.com/pingcap/errors v0.11.0 - github.com/pingcap/kvproto v0.0.0-20181206061346-54cf0a0dfe55 // indirect - github.com/pingcap/parser v0.0.0-20181205023950-c563561800a2 - github.com/pingcap/tidb v0.0.0-20181206093020-98c72c6a8ee3 - github.com/pingcap/tidb-tools v2.1.3-0.20190115072802-b674be072353+incompatible - github.com/pingcap/tipb v0.0.0-20181126132056-a7fd2aaa9719 // indirect + github.com/pingcap/parser v0.0.0-20190219084020-362712b5ab93 + github.com/pingcap/tidb v0.0.0-20190219092241-45ee2058969c + github.com/pingcap/tidb-tools v2.1.3-0.20190218062145-515a74c8fb26+incompatible github.com/prometheus/client_golang v0.9.1 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a // indirect @@ -31,9 +29,8 @@ require ( github.com/soheilhy/cmux v0.1.4 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect - github.com/stretchr/testify v1.3.0 // indirect golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc // indirect - golang.org/x/net v0.0.0-20181201002055-351d144fa1fc + golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e golang.org/x/sys v0.0.0-20190116161447-11f53e031339 google.golang.org/grpc v1.17.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/go.sum b/go.sum index 1aed3b88b8..a32d28efdc 100644 --- a/go.sum +++ b/go.sum @@ -58,8 +58,8 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9 h1:EGUd+AQfZoi1OwZAoqekLbl4kq6tafFtKQSiN8nL21Y= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg= @@ -115,31 +115,32 @@ github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7l github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPAM7gywcUKa0z58gczVrnG6TQnAQ= -github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 h1:04yuCf5NMvLU8rB2m4Qs3rynH7EYpMno3lHkewIOdMo= +github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/kvproto v0.0.0-20181105061835-1b5d69cd1d26/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk= -github.com/pingcap/kvproto v0.0.0-20181206061346-54cf0a0dfe55 h1:0vbRC39OU0Ep2GFgV5BwZ1u3viC5uk8yb3c07rLRMvY= -github.com/pingcap/kvproto v0.0.0-20181206061346-54cf0a0dfe55/go.mod h1:Ja9XPjot9q4/3JyCZodnWDGNXt4pKemhIYCvVJM7P24= -github.com/pingcap/parser v0.0.0-20181204031237-721bcaad1aeb/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/parser v0.0.0-20181205023950-c563561800a2 h1:MZ88gOZZLBNL0RLFFxHfIG0233zhzIq+RyIJTVkULfo= -github.com/pingcap/parser v0.0.0-20181205023950-c563561800a2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/kvproto v0.0.0-20190131052532-7e329e0c9e32 h1:9uwqk2DvsAKImRKYAjERMuIf5ZiCcNFhaFhgFRXw7X0= +github.com/pingcap/kvproto v0.0.0-20190131052532-7e329e0c9e32/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= +github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7 h1:kOHAMalwF69bJrtWrOdVaCSvZjLucrJhP4NQKIu6uM4= +github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= +github.com/pingcap/parser v0.0.0-20190218033509-9545f168ae97/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190219084020-362712b5ab93 h1:jRnleVTpcisLEHNRYHBu4ilwEPyUkslJJfhYNEQlz2g= +github.com/pingcap/parser v0.0.0-20190219084020-362712b5ab93/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE= github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= -github.com/pingcap/tidb v0.0.0-20181206093020-98c72c6a8ee3 h1:VW18UjQwAwsvod4oq/2bigmniV8a2dss44LRADF/UFg= -github.com/pingcap/tidb v0.0.0-20181206093020-98c72c6a8ee3/go.mod h1:ePvdDXDo3CWkhAlwgnVgj7w3PkpXYum72/fHVng0wQI= -github.com/pingcap/tidb-tools v0.0.0-20181112132202-4860a0d5de03/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tidb-tools v2.1.3-0.20190115072802-b674be072353+incompatible h1:pMaTt63rmQ1wDZo2E3Iyci+KqP2C5yKvfrtRwWSx5Rs= -github.com/pingcap/tidb-tools v2.1.3-0.20190115072802-b674be072353+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20170310053819-1043caee48da/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pingcap/tipb v0.0.0-20181126132056-a7fd2aaa9719 h1:MEHhwNcWuuoJJvYxMaWIXFp+BvK/WORPiiwmSOS0cA4= -github.com/pingcap/tipb v0.0.0-20181126132056-a7fd2aaa9719/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pingcap/tidb v0.0.0-20190219092241-45ee2058969c h1:1BlUrIxyx/nJ2C3Cr5AbFHjeGloqZgfp7dYeojuP2FI= +github.com/pingcap/tidb v0.0.0-20190219092241-45ee2058969c/go.mod h1:UX+3xES7AhBMvwxZwOqvMHScn5bYN+EIA6LsUp+zCvA= +github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tidb-tools v2.1.3-0.20190218062145-515a74c8fb26+incompatible h1:stzSBztIf9rHk3yOYghbFCuQ+CbRypH/h9fnTEB+eaU= +github.com/pingcap/tidb-tools v2.1.3-0.20190218062145-515a74c8fb26+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7 h1:wnjdQRhybddDesBVBKyOLUPgDaOFdtqA92pduBgWvVQ= +github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -220,33 +221,38 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190116161447-11f53e031339 h1:g/Jesu8+QLnA0CPzF3E1pURg0Byr7i6jLoX5sqjcAh0= golang.org/x/sys v0.0.0-20190116161447-11f53e031339/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.0.0-20171214130843-f21a4dfb5e38/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181008205924-a2b3f7f249e9/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0 h1:iRpjPej1fPzmfoBhMFkp3HdqzF+ytPmAwiQhJGV0zGw= +golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f h1:FU37niK8AQ59mHcskRyQL7H0ErSeNh650vdcj8HqdSI= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 h1:9oFlwfEGIvmxXTcY53ygNyxIQtWciRHjrnUvZJCYXYU= +google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= diff --git a/loader/convert_data.go b/loader/convert_data.go index 19e26c8e8f..54acf674e1 100644 --- a/loader/convert_data.go +++ b/loader/convert_data.go @@ -22,6 +22,7 @@ import ( "strings" "unsafe" + "github.com/pingcap/dm/pkg/ddl" "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -217,7 +218,7 @@ func parseTable(r *router.Table, schema, table, file string) (*tableInfo, error) return nil, errors.Annotatef(err, "read table info from file %s", file) } - stmts, err := parser.New().Parse(string(statement), "", "") + stmts, err := ddl.Parse(parser.New(), string(statement), "", "") if err != nil { return nil, errors.Annotatef(err, "parser statement %s", statement) } diff --git a/pkg/ddl/common.go b/pkg/ddl/common.go new file mode 100644 index 0000000000..87bbb81a37 --- /dev/null +++ b/pkg/ddl/common.go @@ -0,0 +1,35 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package ddl + +import ( + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/parser" + "github.com/pingcap/parser/ast" +) + +// Parse wraps parser.Parse(), makes `parser` suitable for dm +func Parse(p *parser.Parser, sql, charset, collation string) (stmt []ast.StmtNode, err error) { + stmts, warnings, err := p.Parse(sql, charset, collation) + + for _, warning := range warnings { + log.Warnf("warning: parsing sql %s:%v", sql, warning) + } + + if err != nil { + log.Errorf("error: parsing sql %s:%v", sql, err) + } + + return stmts, err +} diff --git a/relay/util.go b/relay/util.go index 6669a6c41c..9330ca8a8a 100644 --- a/relay/util.go +++ b/relay/util.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/parser/ast" "github.com/siddontang/go-mysql/replication" + "github.com/pingcap/dm/pkg/ddl" "github.com/pingcap/dm/pkg/utils" ) @@ -29,7 +30,7 @@ func checkIsDDL(sql string, p *parser.Parser) bool { sql = utils.TrimCtrlChars(sql) // if parse error, treat it as not a DDL - stmts, err := p.Parse(sql, "", "") + stmts, err := ddl.Parse(p, sql, "", "") if err != nil || len(stmts) == 0 { return false } diff --git a/syncer/ddl.go b/syncer/ddl.go index 0d55c50279..e02c31f716 100644 --- a/syncer/ddl.go +++ b/syncer/ddl.go @@ -16,6 +16,7 @@ package syncer import ( "fmt" + ddlpkg "github.com/pingcap/dm/pkg/ddl" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/errors" "github.com/pingcap/parser" @@ -67,7 +68,7 @@ func (s *Syncer) parseDDLSQL(sql string, p *parser.Parser, schema string) (resul // We use Parse not ParseOneStmt here, because sometimes we got a commented out ddl which can't be parsed // by ParseOneStmt(it's a limitation of tidb parser.) - stmts, err := p.Parse(sql, "", "") + stmts, err := ddlpkg.Parse(p, sql, "", "") if err != nil { // log error rather than fatal, so other defer can be executed log.Errorf(IncompatibleDDLFormat, sql) From 060638e83d0bb1a4d02b776e673f6a99a454cfd9 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Tue, 19 Feb 2019 18:09:10 +0800 Subject: [PATCH 02/20] *: update syncer code --- pkg/ddl/common.go | 245 +++++++++++++++++++- syncer/ast.go | 516 ------------------------------------------- syncer/ast_test.go | 351 ----------------------------- syncer/ddl.go | 240 ++------------------ syncer/inject_sql.go | 3 +- syncer/online_ddl.go | 5 + syncer/syncer.go | 2 +- 7 files changed, 266 insertions(+), 1096 deletions(-) delete mode 100644 syncer/ast.go delete mode 100644 syncer/ast_test.go diff --git a/pkg/ddl/common.go b/pkg/ddl/common.go index 87bbb81a37..d1f19cc240 100644 --- a/pkg/ddl/common.go +++ b/pkg/ddl/common.go @@ -14,9 +14,15 @@ package ddl import ( + "bytes" + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/format" + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb-tools/pkg/filter" ) // Parse wraps parser.Parse(), makes `parser` suitable for dm @@ -31,5 +37,242 @@ func Parse(p *parser.Parser, sql, charset, collation string) (stmt []ast.StmtNod log.Errorf("error: parsing sql %s:%v", sql, err) } - return stmts, err + return stmts, errors.Trace(err) +} + +// FetchDDLTableNames returns table names in ddl +// the result contains [tableName] excepted create table like and rename table +// for `create table like` DDL, result contains [sourceTableName, sourceRefTableName] +// for rename table ddl, result contains [targetOldTableName, sourceNewTableName] +func FetchDDLTableNames(schema string, stmt ast.StmtNode) ([]*filter.Table, error) { + var res []*filter.Table + switch v := stmt.(type) { + case *ast.CreateDatabaseStmt: + res = append(res, genTableName(v.Name, "")) + case *ast.DropDatabaseStmt: + res = append(res, genTableName(v.Name, "")) + case *ast.CreateTableStmt: + res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) + if v.ReferTable != nil { + res = append(res, genTableName(v.ReferTable.Schema.O, v.ReferTable.Name.O)) + } + case *ast.DropTableStmt: + if len(v.Tables) != 1 { + return res, errors.Errorf("drop table with multiple tables, may resovle ddl sql failed") + } + res = append(res, genTableName(v.Tables[0].Schema.O, v.Tables[0].Name.O)) + case *ast.TruncateTableStmt: + res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) + case *ast.AlterTableStmt: + res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) + if v.Specs[0].NewTable != nil { + res = append(res, genTableName(v.Specs[0].NewTable.Schema.O, v.Specs[0].NewTable.Name.O)) + } + case *ast.RenameTableStmt: + res = append(res, genTableName(v.OldTable.Schema.O, v.OldTable.Name.O)) + res = append(res, genTableName(v.NewTable.Schema.O, v.NewTable.Name.O)) + case *ast.CreateIndexStmt: + res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) + case *ast.DropIndexStmt: + res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) + default: + return res, errors.Errorf("unkown type ddl %s", stmt) + } + + for i := range res { + if res[i].Schema == "" { + res[i].Schema = schema + } + } + + return res, nil +} + +// RenameDDLTable renames table names in ddl by given `targetNames` +func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string, error) { + switch v := stmt.(type) { + case *ast.CreateDatabaseStmt: + v.Name = targetTableNames[0].Schema + + case *ast.DropDatabaseStmt: + v.Name = targetTableNames[0].Schema + + case *ast.CreateTableStmt: + v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema) + v.Table.Name = model.NewCIStr(targetTableNames[0].Name) + + if v.ReferTable != nil { + v.ReferTable.Schema = model.NewCIStr(targetTableNames[1].Schema) + v.ReferTable.Name = model.NewCIStr(targetTableNames[1].Name) + } + + case *ast.DropTableStmt: + if len(v.Tables) > 0 { + return "", errors.New("not allow operation: delete multiple tables in one statement") + } + + v.Tables[0].Schema = model.NewCIStr(targetTableNames[0].Schema) + v.Tables[0].Name = model.NewCIStr(targetTableNames[0].Name) + + case *ast.TruncateTableStmt: + v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema) + v.Table.Name = model.NewCIStr(targetTableNames[0].Name) + + case *ast.DropIndexStmt: + v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema) + v.Table.Name = model.NewCIStr(targetTableNames[0].Name) + case *ast.CreateIndexStmt: + v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema) + v.Table.Name = model.NewCIStr(targetTableNames[0].Name) + case *ast.RenameTableStmt: + if len(v.TableToTables) > 0 { + return "", errors.New("not allow operation: rename multiple tables in one statement") + } + + v.TableToTables[0].OldTable.Schema = model.NewCIStr(targetTableNames[0].Schema) + v.TableToTables[0].OldTable.Name = model.NewCIStr(targetTableNames[0].Name) + v.TableToTables[0].NewTable.Schema = model.NewCIStr(targetTableNames[1].Schema) + v.TableToTables[0].NewTable.Name = model.NewCIStr(targetTableNames[1].Name) + + case *ast.AlterTableStmt: + if len(v.Specs) > 0 { + return "", errors.New("not allow operation: rename multiple tables in one statement") + } + + v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema) + v.Table.Name = model.NewCIStr(targetTableNames[0].Name) + + default: + return "", errors.Errorf("unkown type ddl %+v", stmt) + } + + var b []byte + bf := bytes.NewBuffer(b) + err := stmt.Restore(&format.RestoreCtx{ + Flags: format.DefaultRestoreFlags, + In: bf, + }) + if err != nil { + return "", errors.Annotate(err, "restore ast node") + } + + return bf.String(), nil +} + +// SplitDDL split multiple operations in one DDL statement into multiple DDL statements +func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { + var ( + b []byte + schemaName = model.NewCIStr(schema) // fill schema name + bf = bytes.NewBuffer(b) + ctx = &format.RestoreCtx{ + Flags: format.DefaultRestoreFlags, + In: bf, + } + ) + + switch v := stmt.(type) { + case *ast.CreateDatabaseStmt: + v.IfNotExists = true + case *ast.DropDatabaseStmt: + v.IfExists = true + case *ast.DropTableStmt: + v.IfExists = true + for _, t := range v.Tables { + if t.Schema.O == "" { + t.Schema = schemaName + } + } + case *ast.CreateTableStmt: + v.IfNotExists = true + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + + if v.ReferTable != nil && v.ReferTable.Schema.O == "" { + v.ReferTable.Schema = schemaName + } + case *ast.TruncateTableStmt: + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + case *ast.DropIndexStmt: + v.IfExists = true + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + case *ast.CreateIndexStmt: + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + case *ast.RenameTableStmt: + t2ts := v.TableToTables + for _, t2t := range t2ts { + if t2t.OldTable.Schema.O == "" { + t2t.OldTable.Schema = schemaName + } + if t2t.NewTable.Schema.O == "" { + t2t.NewTable.Schema = schemaName + } + + v.TableToTables = []*ast.TableToTable{t2t} + + bf.Reset() + err = stmt.Restore(ctx) + if err != nil { + return nil, errors.Annotate(err, "restore ast node") + } + + sqls = append(sqls, bf.String()) + } + v.TableToTables = t2ts + + return sqls, nil + case *ast.AlterTableStmt: + specs := v.Specs + + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + + for _, spec := range specs { + if spec.Tp == ast.AlterTableRenameTable { + if spec.NewTable.Schema.O == "" { + spec.NewTable.Schema = schemaName + } + } + + v.Specs = []*ast.AlterTableSpec{spec} + + bf.Reset() + err = stmt.Restore(ctx) + if err != nil { + return nil, errors.Annotate(err, "restore ast node") + } + sqls = append(sqls, bf.String()) + + if spec.Tp == ast.AlterTableRenameTable { + v.Table = spec.NewTable + } + } + v.Specs = specs + + return sqls, nil + + default: + return nil, errors.Errorf("unkown type ddl %+v", stmt) + } + + bf.Reset() + err = stmt.Restore(ctx) + if err != nil { + return nil, errors.Annotate(err, "restore ast node") + } + sqls = append(sqls, bf.String()) + + return sqls, nil +} + +func genTableName(schema string, table string) *filter.Table { + return &filter.Table{Schema: schema, Name: table} } diff --git a/syncer/ast.go b/syncer/ast.go deleted file mode 100644 index c07700315f..0000000000 --- a/syncer/ast.go +++ /dev/null @@ -1,516 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncer - -import ( - "bytes" - "fmt" - "strings" - - "github.com/pingcap/dm/pkg/log" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/charset" - "github.com/pingcap/parser/model" - "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb/types" - _ "github.com/pingcap/tidb/types/parser_driver" // use value expression impl in TiDB -) - -func defaultValueToSQL(opt *ast.ColumnOption) string { - var value string - - switch expr := opt.Expr.(type) { - case *ast.UnaryOperationExpr: - var str bytes.Buffer - expr.Format(&str) - value = str.String() - - case *ast.FuncCallExpr: - value = expr.FnName.O - - case ast.ValueExpr: - datum := opt.Expr.(ast.ValueExpr) - buf := new(bytes.Buffer) - datum.Format(buf) - value = buf.String() - if len(value) > 1 && value[0] == '"' && value[len(value)-1] == '"' { - value = formatStringValue(value[1 : len(value)-1]) - } - - default: - log.Errorf("unhandle expression type: %v", opt.Expr) - value = "NULL" - } - - return fmt.Sprintf(" DEFAULT %s", value) -} - -func formatStringValue(s string) string { - if s == "" { - return "''" - } - return fmt.Sprintf("'%s'", escapeSingleQuote(s)) -} - -// for change/modify column use only. -func fieldTypeToSQL(ft *types.FieldType) string { - strs := []string{ft.CompactStr()} - log.Debugf("tp %v, flag %v, flen %v, decimal %v, charset %v, collate %v, strs %v", - ft.Tp, ft.Flag, ft.Flen, ft.Decimal, ft.Charset, ft.Collate, strs) - if mysql.HasUnsignedFlag(ft.Flag) { - strs = append(strs, "UNSIGNED") - } - if mysql.HasZerofillFlag(ft.Flag) { - strs = append(strs, "ZEROFILL") - } - - if mysql.HasBinaryFlag(ft.Flag) && (ft.Charset != charset.CharsetBin || (!types.IsTypeChar(ft.Tp) && !types.IsTypeBlob(ft.Tp))) { - strs = append(strs, "BINARY") - } - - return strings.Join(strs, " ") -} - -// for add column use only. Work likes FieldType.String(), but bug free(?) -func fullFieldTypeToSQL(ft *types.FieldType) string { - sql := fieldTypeToSQL(ft) - strs := strings.Split(sql, " ") - - if types.IsTypeChar(ft.Tp) || types.IsTypeBlob(ft.Tp) { - if ft.Charset != "" && ft.Charset != charset.CharsetBin { - strs = append(strs, fmt.Sprintf("CHARACTER SET %s", ft.Charset)) - } - if ft.Collate != "" && ft.Collate != charset.CharsetBin { - strs = append(strs, fmt.Sprintf("COLLATE %s", ft.Collate)) - } - } - - return strings.Join(strs, " ") -} - -// FIXME: tidb's AST is error-some to handle more condition -func columnOptionsToSQL(options []*ast.ColumnOption) string { - sql := "" - for _, opt := range options { - switch opt.Tp { - case ast.ColumnOptionNotNull: - sql += " NOT NULL" - case ast.ColumnOptionNull: - sql += " NULL" - case ast.ColumnOptionDefaultValue: - sql += defaultValueToSQL(opt) - case ast.ColumnOptionAutoIncrement: - sql += " AUTO_INCREMENT" - case ast.ColumnOptionUniqKey: - sql += " UNIQUE KEY" - case ast.ColumnOptionPrimaryKey: - sql += " PRIMARY KEY" - case ast.ColumnOptionComment: - comment := opt.Expr.(ast.ValueExpr).GetDatumString() - comment = escapeSingleQuote(comment) - sql += fmt.Sprintf(" COMMENT '%s'", comment) - case ast.ColumnOptionOnUpdate: // For Timestamp and Datetime only. - sql += " ON UPDATE CURRENT_TIMESTAMP" - case ast.ColumnOptionFulltext: - panic("not implemented yet") - default: - panic("not implemented yet") - } - } - - return sql -} - -func escapeName(name string) string { - return strings.Replace(name, "`", "``", -1) -} - -func tableNameToSQL(tbl *ast.TableName) string { - sql := "" - if tbl.Schema.O != "" { - sql += fmt.Sprintf("`%s`.", tbl.Schema.O) - } - sql += fmt.Sprintf("`%s`", tbl.Name.O) - return sql -} - -func columnNameToSQL(name *ast.ColumnName) string { - sql := "" - if name.Schema.O != "" { - sql += fmt.Sprintf("`%s`.", escapeName(name.Schema.O)) - } - if name.Table.O != "" { - sql += fmt.Sprintf("`%s`.", escapeName(name.Table.O)) - } - sql += fmt.Sprintf("`%s`", escapeName(name.Name.O)) - return sql -} - -func indexColNameToSQL(name *ast.IndexColName) string { - sql := columnNameToSQL(name.Column) - if name.Length != types.UnspecifiedLength { - sql += fmt.Sprintf(" (%d)", name.Length) - } - return sql -} - -func constraintKeysToSQL(keys []*ast.IndexColName) string { - if len(keys) == 0 { - panic("unreachable") - } - sql := "" - for i, indexColName := range keys { - if i == 0 { - sql += "(" - } - sql += indexColNameToSQL(indexColName) - if i != len(keys)-1 { - sql += ", " - } - } - sql += ")" - return sql -} - -func referenceDefToSQL(refer *ast.ReferenceDef) string { - sql := fmt.Sprintf("%s ", tableNameToSQL(refer.Table)) - sql += constraintKeysToSQL(refer.IndexColNames) - if refer.OnDelete != nil && refer.OnDelete.ReferOpt != ast.ReferOptionNoOption { - sql += fmt.Sprintf(" ON DELETE %s", refer.OnDelete.ReferOpt) - } - if refer.OnUpdate != nil && refer.OnUpdate.ReferOpt != ast.ReferOptionNoOption { - sql += fmt.Sprintf(" ON UPDATE %s", refer.OnUpdate.ReferOpt) - } - return sql -} - -func indexTypeToSQL(opt *ast.IndexOption) string { - // opt can be nil. - if opt == nil { - return "" - } - switch opt.Tp { - case model.IndexTypeBtree: - return "USING BTREE " - case model.IndexTypeHash: - return "USING HASH " - default: - // nothing to do - return "" - } -} - -func constraintToSQL(constraint *ast.Constraint) string { - sql := "" - switch constraint.Tp { - case ast.ConstraintKey, ast.ConstraintIndex: - sql += "ADD INDEX " - if constraint.Name != "" { - sql += fmt.Sprintf("`%s` ", escapeName(constraint.Name)) - } - sql += indexTypeToSQL(constraint.Option) - sql += constraintKeysToSQL(constraint.Keys) - sql += indexOptionToSQL(constraint.Option) - - case ast.ConstraintUniq, ast.ConstraintUniqKey, ast.ConstraintUniqIndex: - sql += "ADD CONSTRAINT " - if constraint.Name != "" { - sql += fmt.Sprintf("`%s` ", escapeName(constraint.Name)) - } - sql += "UNIQUE INDEX " - sql += indexTypeToSQL(constraint.Option) - sql += constraintKeysToSQL(constraint.Keys) - sql += indexOptionToSQL(constraint.Option) - - case ast.ConstraintForeignKey: - sql += "ADD CONSTRAINT " - if constraint.Name != "" { - sql += fmt.Sprintf("`%s` ", escapeName(constraint.Name)) - } - sql += "FOREIGN KEY " - sql += constraintKeysToSQL(constraint.Keys) - sql += " REFERENCES " - sql += referenceDefToSQL(constraint.Refer) - - case ast.ConstraintPrimaryKey: - sql += "ADD CONSTRAINT " - if constraint.Name != "" { - sql += fmt.Sprintf("`%s` ", escapeName(constraint.Name)) - } - sql += "PRIMARY KEY " - sql += indexTypeToSQL(constraint.Option) - sql += constraintKeysToSQL(constraint.Keys) - sql += indexOptionToSQL(constraint.Option) - - case ast.ConstraintFulltext: - sql += "ADD FULLTEXT INDEX " - if constraint.Name != "" { - sql += fmt.Sprintf("`%s` ", escapeName(constraint.Name)) - } - sql += constraintKeysToSQL(constraint.Keys) - sql += indexOptionToSQL(constraint.Option) - - default: - panic("not implemented yet") - } - return sql -} - -func positionToSQL(pos *ast.ColumnPosition) string { - var sql string - switch pos.Tp { - case ast.ColumnPositionNone: - case ast.ColumnPositionFirst: - sql = " FIRST" - case ast.ColumnPositionAfter: - colName := pos.RelativeColumn.Name.O - sql = fmt.Sprintf(" AFTER `%s`", escapeName(colName)) - default: - panic("unreachable") - } - return sql -} - -// Convert constraint indexoption to sql. Currently only support comment. -func indexOptionToSQL(option *ast.IndexOption) string { - if option == nil { - return "" - } - - if option.Comment != "" { - return fmt.Sprintf(" COMMENT '%s'", escapeSingleQuote(option.Comment)) - } - - return "" -} - -func tableOptionsToSQL(options []*ast.TableOption) string { - sql := "" - if len(options) == 0 { - return sql - } - - for _, opt := range options { - switch opt.Tp { - case ast.TableOptionEngine: - if opt.StrValue == "" { - sql += fmt.Sprintf(" ENGINE = ''") - } else { - sql += fmt.Sprintf(" ENGINE = %s", opt.StrValue) - } - - case ast.TableOptionCharset: - sql += fmt.Sprintf(" CHARACTER SET = %s", opt.StrValue) - - case ast.TableOptionCollate: - sql += fmt.Sprintf(" COLLATE = %s", opt.StrValue) - - case ast.TableOptionAutoIncrement: - sql += fmt.Sprintf(" AUTO_INCREMENT = %d", opt.UintValue) - - case ast.TableOptionComment: - sql += fmt.Sprintf(" COMMENT '%s'", escapeSingleQuote(opt.StrValue)) - - case ast.TableOptionAvgRowLength: - sql += fmt.Sprintf(" AVG_ROW_LENGTH = %d", opt.UintValue) - - case ast.TableOptionCheckSum: - sql += fmt.Sprintf(" CHECKSUM = %d", opt.UintValue) - - case ast.TableOptionCompression: - // In TiDB parser.y, the rule is "COMPRESSION" EqOpt Identifier. No single quote here. - sql += fmt.Sprintf(" COMPRESSION = '%s'", escapeSingleQuote(opt.StrValue)) - - case ast.TableOptionConnection: - sql += fmt.Sprintf(" CONNECTION = '%s'", escapeSingleQuote(opt.StrValue)) - - case ast.TableOptionPassword: - sql += fmt.Sprintf(" PASSWORD = '%s'", escapeSingleQuote(opt.StrValue)) - - case ast.TableOptionKeyBlockSize: - sql += fmt.Sprintf(" KEY_BLOCK_SIZE = %d", opt.UintValue) - - case ast.TableOptionMaxRows: - sql += fmt.Sprintf(" MAX_ROWS = %d", opt.UintValue) - - case ast.TableOptionMinRows: - sql += fmt.Sprintf(" MIN_ROWS = %d", opt.UintValue) - - case ast.TableOptionDelayKeyWrite: - sql += fmt.Sprintf(" DELAY_KEY_WRITE = %d", opt.UintValue) - - case ast.TableOptionRowFormat: - sql += fmt.Sprintf(" ROW_FORMAT = %s", formatRowFormat(opt.UintValue)) - - case ast.TableOptionStatsPersistent: - // Since TiDB doesn't support this feature, we just give a default value. - sql += " STATS_PERSISTENT = DEFAULT" - default: - panic("unreachable") - } - - } - - // trim prefix space - sql = strings.TrimPrefix(sql, " ") - return sql -} - -func escapeSingleQuote(str string) string { - return strings.Replace(str, "'", "''", -1) -} - -func formatRowFormat(rf uint64) string { - var s string - switch rf { - case ast.RowFormatDefault: - s = "DEFAULT" - case ast.RowFormatDynamic: - s = "DYNAMIC" - case ast.RowFormatFixed: - s = "FIXED" - case ast.RowFormatCompressed: - s = "COMPRESSED" - case ast.RowFormatRedundant: - s = "REDUNDANT" - case ast.RowFormatCompact: - s = "COMPACT" - default: - panic("unreachable") - } - return s -} - -func columnToSQL(typeDef string, newColumn *ast.ColumnDef) string { - return fmt.Sprintf("%s %s%s", columnNameToSQL(newColumn.Name), typeDef, columnOptionsToSQL(newColumn.Options)) -} - -func alterTableSpecToSQL(spec *ast.AlterTableSpec, ntable *ast.TableName) []string { - var ( - suffixes []string - suffix string - ) - - switch spec.Tp { - case ast.AlterTableOption: - suffix += tableOptionsToSQL(spec.Options) - suffixes = append(suffixes, suffix) - - case ast.AlterTableAddColumns: - for _, newColumn := range spec.NewColumns { - suffix = "" - typeDef := fullFieldTypeToSQL(newColumn.Tp) - suffix += fmt.Sprintf("ADD COLUMN %s", columnToSQL(typeDef, newColumn)) - if spec.Position != nil { - suffix += positionToSQL(spec.Position) - } - suffixes = append(suffixes, suffix) - } - - case ast.AlterTableDropColumn: - suffix += fmt.Sprintf("DROP COLUMN %s", columnNameToSQL(spec.OldColumnName)) - suffixes = append(suffixes, suffix) - - case ast.AlterTableDropIndex: - suffix += fmt.Sprintf("DROP INDEX `%s`", escapeName(spec.Name)) - suffixes = append(suffixes, suffix) - - case ast.AlterTableAddConstraint: - suffix += constraintToSQL(spec.Constraint) - suffixes = append(suffixes, suffix) - - case ast.AlterTableDropForeignKey: - suffix += fmt.Sprintf("DROP FOREIGN KEY `%s`", escapeName(spec.Name)) - suffixes = append(suffixes, suffix) - - case ast.AlterTableModifyColumn: - // TiDB doesn't support alter table modify column charset and collation. - typeDef := fieldTypeToSQL(spec.NewColumns[0].Tp) - suffix += fmt.Sprintf("MODIFY COLUMN %s", columnToSQL(typeDef, spec.NewColumns[0])) - if spec.Position != nil { - suffix += positionToSQL(spec.Position) - } - suffixes = append(suffixes, suffix) - - // FIXME: should support [FIRST|AFTER col_name], but tidb parser not support this currently. - case ast.AlterTableChangeColumn: - // TiDB doesn't support alter table change column charset and collation. - typeDef := fieldTypeToSQL(spec.NewColumns[0].Tp) - suffix += "CHANGE COLUMN " - suffix += fmt.Sprintf("%s %s", - columnNameToSQL(spec.OldColumnName), - columnToSQL(typeDef, spec.NewColumns[0])) - if spec.Position != nil { - suffix += positionToSQL(spec.Position) - } - suffixes = append(suffixes, suffix) - - case ast.AlterTableRenameTable: - *ntable = *spec.NewTable - suffix += fmt.Sprintf("RENAME TO %s", tableNameToSQL(spec.NewTable)) - suffixes = append(suffixes, suffix) - - case ast.AlterTableAlterColumn: - suffix += fmt.Sprintf("ALTER COLUMN %s ", columnNameToSQL(spec.NewColumns[0].Name)) - if options := spec.NewColumns[0].Options; options != nil { - suffix += fmt.Sprintf("SET%s", defaultValueToSQL(options[0])) - } else { - suffix += "DROP DEFAULT" - } - suffixes = append(suffixes, suffix) - - case ast.AlterTableDropPrimaryKey: - suffix += "DROP PRIMARY KEY" - suffixes = append(suffixes, suffix) - - case ast.AlterTableLock: - // just ignore it - case ast.AlterTableRenameIndex: - suffix := fmt.Sprintf("RENAME INDEX %s TO %s", indexNameToSQL(spec.FromKey), indexNameToSQL(spec.ToKey)) - suffixes = append(suffixes, suffix) - - default: - } - return suffixes -} - -func indexNameToSQL(name model.CIStr) string { - return fmt.Sprintf("`%s`", escapeName(name.String())) -} - -func alterTableStmtToSQL(stmt *ast.AlterTableStmt, ntable *ast.TableName) []string { - var ( - sqls []string - prefix string - ) - if ntable.Name.O != "" { - prefix = fmt.Sprintf("ALTER TABLE %s ", tableNameToSQL(ntable)) - } else { - prefix = fmt.Sprintf("ALTER TABLE %s ", tableNameToSQL(stmt.Table)) - } - - if len(stmt.Specs) != 1 { - log.Warnf("[syncer] statement specs length != 1, %+v", stmt.Specs) - } - - for _, spec := range stmt.Specs { - log.Infof("spec %+v", spec) - for _, suffix := range alterTableSpecToSQL(spec, ntable) { - sqls = append(sqls, prefix+suffix) - } - } - - log.Debugf("alter table stmt to sql:%v", sqls) - return sqls -} diff --git a/syncer/ast_test.go b/syncer/ast_test.go deleted file mode 100644 index e8a360d2d9..0000000000 --- a/syncer/ast_test.go +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncer - -import ( - "fmt" - - . "github.com/pingcap/check" - - "github.com/pingcap/dm/pkg/utils" -) - -type testCase struct { - sql string - wantSqls []string - wantErr bool -} - -func (s *testSyncerSuite) TestResolveDDLSQL(c *C) { - s.testNonDDL(c) - s.testComments(c) - - // drop table - s.testDropTable(c) - - s.testFailedCases(c) - s.testCreateIndex(c) - - // alter table - s.testAlterTableOption(c) - s.testAlterTableAddColumn(c) - s.testAlterTableDropColumn(c) - s.testAlterTableDropIndex(c) - s.testAlterTableAddConstraint(c) - s.testAlterTableDropForeignKey(c) - s.testAlterTableModifyColumn(c) - s.testAlterTableChangeColumn(c) - s.testAlterTableRenameTable(c) - s.testAlterTableAlterColumn(c) - s.testAlterTableDropPrimaryKey(c) - s.testAlterTableLock(c) - s.testAlterTableConvert(c) -} - -func (s *testSyncerSuite) testNonDDL(c *C) { - tests := []testCase{ - {"/* rds internal mark */ GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, RELOAD, PROCESS, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER on *.* to 'username'@'%' identified by password '*ddasdsadsadsadsd' with grant option", nil, false}, - } - - s.run(c, tests) -} - -func (s *testSyncerSuite) testComments(c *C) { - tests := []testCase{ - {`-- create database foo;`, nil, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testDropTable(c *C) { - tests := []testCase{ - {"drop table `foo`.`bar`", []string{"DROP TABLE `foo`.`bar`"}, false}, - {"drop table if exists `foo`.`bar`", []string{"DROP TABLE IF EXISTS `foo`.`bar`"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testFailedCases(c *C) { - tests := []testCase{ - // cases parse failed and won't be supported in the near future - // {"", nil, false}, // tidb not support fulltext index - {"alter table bar ADD FULLTEXT INDEX `fulltext` (`name`) WITH PARSER ngram", []string{"alter table bar ADD FULLTEXT INDEX `fulltext` (`name`) WITH PARSER ngram"}, true}, // ditto - {"alter table bar ADD SPATIAL INDEX (`g`)", []string{"alter table bar ADD SPATIAL INDEX (`g`)"}, true}, // tidb not support spatial index - - // cases parse failed and should be supported in the near future - // {"ALTER TABLE bar ENABLE KEYS, DISABLE KEYS", []string{"ALTER TABLE `bar` ENABLE KEYS", "ALTER TABLE `bar` DISABLE KEYS"}, false}, - - // tidb not support ORDER BY. - {"alter table bar ORDER BY id1, id2", []string{"alter table bar ORDER BY id1, id2"}, true}, - // tidb not support ON UPDATE CASCADE ON DELETE RESTRICT - {"alter table bar add index (`name`), add FOREIGN KEY (product_category, product_id) REFERENCES product(category, id) ON UPDATE CASCADE ON DELETE RESTRICT", []string{"alter table bar add index (`name`), add FOREIGN KEY (product_category, product_id) REFERENCES product(category, id) ON UPDATE CASCADE ON DELETE RESTRICT"}, true}, - } - - s.run(c, tests) -} - -func (s *testSyncerSuite) testCreateIndex(c *C) { - tests := []testCase{ - {"create indeX id_index ON lookup (id) USING BTREE", []string{"create indeX id_index ON lookup (id) USING BTREE"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableOption(c *C) { - tests := []testCase{ - - {"alter table bar add index (id), character set utf8 collate utf8_bin", []string{"ALTER TABLE `bar` ADD INDEX (`id`)", "ALTER TABLE `bar` CHARACTER SET = utf8 COLLATE = utf8_bin"}, false}, - {"alter table bar add index (id), character set utf8", []string{"ALTER TABLE `bar` ADD INDEX (`id`)", "ALTER TABLE `bar` CHARACTER SET = utf8"}, false}, - {"alter table bar add index (id), collate utf8_bin comment 'bar'", []string{"ALTER TABLE `bar` ADD INDEX (`id`)", "ALTER TABLE `bar` COLLATE = utf8_bin COMMENT 'bar'"}, false}, - - {"alter table bar add index (`c1`), ENGINE = InnoDB COMMENT 'table bar' ROW_FORMAT = COMPRESSED", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` ENGINE = InnoDB COMMENT 'table bar' ROW_FORMAT = COMPRESSED"}, false}, - {"alter table bar add index (`c1`), character set utf8 collate utf8_bin", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` CHARACTER SET = utf8 COLLATE = utf8_bin"}, false}, - {"alter table bar add index (`c1`), auto_increment = 1", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` AUTO_INCREMENT = 1"}, false}, - {"alter table bar add index (`c1`), COMMENT 'bar'", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` COMMENT 'bar'"}, false}, - {"alter table bar add index (`c1`), AVG_ROW_LENGTH = 1024", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` AVG_ROW_LENGTH = 1024"}, false}, - {"alter table bar add index (`c1`), CHECKSUM = 1", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` CHECKSUM = 1"}, false}, - {"alter table bar add index (`c1`), COMPRESSION = 'zlib'", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` COMPRESSION = 'zlib'"}, false}, // - {"alter table bar add index (`c1`), CONNECTION 'mysql://username:password@hostname:port/database/tablename'", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` CONNECTION = 'mysql://username:password@hostname:port/database/tablename'"}, false}, - {"alter table bar add index (`c1`), PASSWORD 'abc123'", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` PASSWORD = 'abc123'"}, false}, - {"alter table bar add index (`c1`), KEY_BLOCK_SIZE = 128", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` KEY_BLOCK_SIZE = 128"}, false}, - {"alter table bar add index (`c1`), MAX_ROWS 2", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` MAX_ROWS = 2"}, false}, - {"alter table bar add index (`c1`), MIN_ROWS 0", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` MIN_ROWS = 0"}, false}, - {"alter table bar add index (`c1`), DELAY_KEY_WRITE = 0", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` DELAY_KEY_WRITE = 0"}, false}, - {"alter table bar add index (`c1`), ROW_FORMAT = COMPACT", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` ROW_FORMAT = COMPACT"}, false}, - {"alter table bar add index (`c1`), ROW_FORMAT = DEFAULT", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` ROW_FORMAT = DEFAULT"}, false}, - {"alter table bar add index (`c1`), ROW_FORMAT = DYNAMIC", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` ROW_FORMAT = DYNAMIC"}, false}, - {"alter table bar add index (`c1`), ROW_FORMAT = COMPRESSED", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` ROW_FORMAT = COMPRESSED"}, false}, - {"alter table bar add index (`c1`), ROW_FORMAT = REDUNDANT", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` ROW_FORMAT = REDUNDANT"}, false}, - {"alter table bar add index (`c1`), ROW_FORMAT = FIXED", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` ROW_FORMAT = FIXED"}, false}, - - {"alter table bar add index (`c1`), STATS_PERSISTENT 1", []string{"ALTER TABLE `bar` ADD INDEX (`c1`)", "ALTER TABLE `bar` STATS_PERSISTENT = DEFAULT"}, false}, - {"alter table bar engine='InnoDB'", []string{"ALTER TABLE `bar` ENGINE = InnoDB"}, false}, - {"alter table bar engine=``", []string{"ALTER TABLE `bar` ENGINE = ''"}, false}, - {"alter table bar engine=''", []string{"ALTER TABLE `bar` ENGINE = ''"}, false}, - {"alter table bar change column `id` `id` int not null, ENGINE=``", []string{"ALTER TABLE `bar` CHANGE COLUMN `id` `id` int(11) NOT NULL", "ALTER TABLE `bar` ENGINE = ''"}, false}, - {"alter table bar engine='', add column id int not null", []string{"ALTER TABLE `bar` ENGINE = ''", "ALTER TABLE `bar` ADD COLUMN `id` int(11) NOT NULL"}, false}, - } - - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableAddColumn(c *C) { - tests := []testCase{ - {"alter table `bar` add column `id` int(11) not null default -1", []string{"ALTER TABLE `bar` ADD COLUMN `id` int(11) NOT NULL DEFAULT -1"}, false}, - {"alter table `bar` add column `id` int(11) not null default +1", []string{"ALTER TABLE `bar` ADD COLUMN `id` int(11) NOT NULL DEFAULT +1"}, false}, - {"alter table `bar` add column `id` int(11) not null default 1", []string{"ALTER TABLE `bar` ADD COLUMN `id` int(11) NOT NULL DEFAULT 1"}, false}, - {"alter table `bar` add column `id` float not null default -1.1", []string{"ALTER TABLE `bar` ADD COLUMN `id` float NOT NULL DEFAULT -1.1"}, false}, - {"alter table `bar` add column `id` float not null default +1.1", []string{"ALTER TABLE `bar` ADD COLUMN `id` float NOT NULL DEFAULT +1.1"}, false}, - {"alter table `bar` add column `id` float not null default 1.1", []string{"ALTER TABLE `bar` ADD COLUMN `id` float NOT NULL DEFAULT 1.1"}, false}, - - {"alter table `bar` add column `id` int(11) unsigned zerofill not null", []string{"ALTER TABLE `bar` ADD COLUMN `id` int(11) UNSIGNED ZEROFILL NOT NULL"}, false}, - {"alter table `bar` add column `id1` int(11) not null primary key comment 'id1', add column id2 int(11) primary key, add id3 int(11) unique, add id4 int(11) unique key", []string{"ALTER TABLE `bar` ADD COLUMN `id1` int(11) NOT NULL PRIMARY KEY COMMENT 'id1'", "ALTER TABLE `bar` ADD COLUMN `id2` int(11) PRIMARY KEY", "ALTER TABLE `bar` ADD COLUMN `id3` int(11) UNIQUE KEY", "ALTER TABLE `bar` ADD COLUMN `id4` int(11) UNIQUE KEY"}, false}, - {"alter table `bar` add column `id1` int(11) not null, add column `id2` int(11) not null default 1", []string{"ALTER TABLE `bar` ADD COLUMN `id1` int(11) NOT NULL", "ALTER TABLE `bar` ADD COLUMN `id2` int(11) NOT NULL DEFAULT 1"}, false}, - {"alter table `bar` add column `id1` int(11) not null, add column `id2` decimal(14,2) not null default 0.00", []string{"ALTER TABLE `bar` ADD COLUMN `id1` int(11) NOT NULL", "ALTER TABLE `bar` ADD COLUMN `id2` decimal(14,2) NOT NULL DEFAULT 0.00"}, false}, - {"alter table `bar` add column `id1` int(11) not null, add column `id2` int(11) not null COMMENT 'this is id2'", []string{"ALTER TABLE `bar` ADD COLUMN `id1` int(11) NOT NULL", "ALTER TABLE `bar` ADD COLUMN `id2` int(11) NOT NULL COMMENT 'this is id2'"}, false}, - {"alter table `bar` add column `id2` int(11) not null first", []string{"ALTER TABLE `bar` ADD COLUMN `id2` int(11) NOT NULL FIRST"}, false}, - {"alter table `bar` add column `id1` int(11) not null, add column `id2` int(11) not null first", []string{"ALTER TABLE `bar` ADD COLUMN `id1` int(11) NOT NULL", "ALTER TABLE `bar` ADD COLUMN `id2` int(11) NOT NULL FIRST"}, false}, - {"alter table `bar` add column `id1` int(11) not null, add column `id2` int(11) not null after `id1`", []string{"ALTER TABLE `bar` ADD COLUMN `id1` int(11) NOT NULL", "ALTER TABLE `bar` ADD COLUMN `id2` int(11) NOT NULL AFTER `id1`"}, false}, - - {"alter table bar add c1 timestamp not null on update current_timestamp, add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` timestamp NOT NULL ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter table bar add c1 timestamp null on update current_timestamp, add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` timestamp NULL ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter table bar add c1 timestamp on update current_timestamp, add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` timestamp ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter table bar add c1 timestamp null default null on update current_timestamp, add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` timestamp NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - - {"alter table bar add c1 timestamp null default 20150606 on update current_timestamp, add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` timestamp NULL DEFAULT 20150606 ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter table bar add c1 timestamp not null default 20150606 on update current_timestamp, add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` timestamp NOT NULL DEFAULT 20150606 ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter table bar add c1 timestamp default 20150606 on update current_timestamp, add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` timestamp DEFAULT 20150606 ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter table bar add c1 timestamp default current_timestamp on update current_timestamp, add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter table bar add c1 timestamp not null default now() on update now(), add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter table bar add c1 varchar(10) DEFAULT '' NOT NULL, add c2 varchar(10) NOT NULL DEFAULT 'foo'", []string{"ALTER TABLE `bar` ADD COLUMN `c1` varchar(10) DEFAULT '' NOT NULL", "ALTER TABLE `bar` ADD COLUMN `c2` varchar(10) NOT NULL DEFAULT 'foo'"}, false}, - {"alter table bar add c1 int(11) not null default 100000000000000, add c2 smallint not null default '100000000000000'", []string{"ALTER TABLE `bar` ADD COLUMN `c1` int(11) NOT NULL DEFAULT 100000000000000", "ALTER TABLE `bar` ADD COLUMN `c2` smallint(6) NOT NULL DEFAULT '100000000000000'"}, false}, - {"alter table bar add c1 enum('','UNO','DUE') NOT NULL default '', add index (c1)", []string{"ALTER TABLE `bar` ADD COLUMN `c1` enum('','UNO','DUE') NOT NULL DEFAULT ''", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter table od_order add column caculating_string varchar(2) CHARACTER SET utf8 COLLATE utf8_general_ci null COMMENT '计费重量体积' after delivery_status, add index (caculating_string)", []string{"ALTER TABLE `od_order` ADD COLUMN `caculating_string` varchar(2) CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '计费重量体积' AFTER `delivery_status`", "ALTER TABLE `od_order` ADD INDEX (`caculating_string`)"}, false}, // https://github.com/pingcap/tidb-enterprise-tools/issues/115 - {"alter table bar add column c1 tinyblob", []string{"ALTER TABLE `bar` ADD COLUMN `c1` tinyblob"}, false}, - {"alter table bar add column c1 blob", []string{"ALTER TABLE `bar` ADD COLUMN `c1` blob"}, false}, - {"alter table bar add column c1 mediumblob", []string{"ALTER TABLE `bar` ADD COLUMN `c1` mediumblob"}, false}, - {"alter table bar add column c1 longblob", []string{"ALTER TABLE `bar` ADD COLUMN `c1` longblob"}, false}, - {"alter table bar add column `id` varchar(20) BINARY not null default ''", []string{"ALTER TABLE `bar` ADD COLUMN `id` varchar(20) BINARY NOT NULL DEFAULT ''"}, false}, - {"alter table bar add column `id` char(20) binary not null default ''", []string{"ALTER TABLE `bar` ADD COLUMN `id` char(20) BINARY NOT NULL DEFAULT ''"}, false}, - {"alter table bar add column `id` text binary", []string{"ALTER TABLE `bar` ADD COLUMN `id` text BINARY"}, false}, - - {"alter table `bar` add column (`id` int(11) unsigned zerofill not null)", []string{"ALTER TABLE `bar` ADD COLUMN `id` int(11) UNSIGNED ZEROFILL NOT NULL"}, false}, - {"alter /* gh-ost */ table `foo`.`bar` add column ( id int unsigned default 0 comment '')", []string{"ALTER TABLE `foo`.`bar` ADD COLUMN `id` int(11) UNSIGNED DEFAULT 0 COMMENT ''"}, false}, - - // add multi-columns in parentheses. - {"alter table bar add column (`id` int unsigned not null, `name` char(20) not null default 'xxx')", []string{"ALTER TABLE `bar` ADD COLUMN `id` int(11) UNSIGNED NOT NULL", "ALTER TABLE `bar` ADD COLUMN `name` char(20) NOT NULL DEFAULT 'xxx'"}, false}, - } - - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableDropColumn(c *C) { - - tests := []testCase{ - {"alter table foo.bar drop a, drop b", []string{"ALTER TABLE `foo`.`bar` DROP COLUMN `a`", "ALTER TABLE `foo`.`bar` DROP COLUMN `b`"}, false}, - } - s.run(c, tests) - -} - -func (s *testSyncerSuite) testAlterTableDropIndex(c *C) { - tests := []testCase{ - {"alter table bar drop key a, drop index b", []string{"ALTER TABLE `bar` DROP INDEX `a`", "ALTER TABLE `bar` DROP INDEX `b`"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableAddConstraint(c *C) { - tests := []testCase{ - {"alter table `bar` add index (`id`)", []string{"ALTER TABLE `bar` ADD INDEX (`id`)"}, false}, - {"alter table `bar` add key (`id`)", []string{"ALTER TABLE `bar` ADD INDEX (`id`)"}, false}, - {"alter table `bar` add index `idx`(`id`, `name`), add index (`name`, `id`) comment 'second index'", []string{"ALTER TABLE `bar` ADD INDEX `idx` (`id`, `name`)", "ALTER TABLE `bar` ADD INDEX (`name`, `id`) COMMENT 'second index'"}, false}, // doubt this. mysql doesn't have ADD CONSTRAINT INDEX syntax - {"alter table `bar` add index `idx`(`id`, `name`), add key (`name`)", []string{"ALTER TABLE `bar` ADD INDEX `idx` (`id`, `name`)", "ALTER TABLE `bar` ADD INDEX (`name`)"}, false}, - - {"alter table bar ADD CONSTRAINT `pri` PRIMARY KEY (`g`), add index (`h`);", []string{"ALTER TABLE `bar` ADD CONSTRAINT `pri` PRIMARY KEY (`g`)", "ALTER TABLE `bar` ADD INDEX (`h`)"}, false}, - {"alter table bar ADD c INT unsigned NOT NULL AUTO_INCREMENT,ADD PRIMARY KEY (c);", []string{"ALTER TABLE `bar` ADD COLUMN `c` int(11) UNSIGNED NOT NULL AUTO_INCREMENT", "ALTER TABLE `bar` ADD CONSTRAINT PRIMARY KEY (`c`)"}, false}, - {"alter table bar ADD index (name), add constraint `u1` unique (`u1`), add unique key (`u2`), add unique index (`u3`);", []string{"ALTER TABLE `bar` ADD INDEX (`name`)", "ALTER TABLE `bar` ADD CONSTRAINT `u1` UNIQUE INDEX (`u1`)", "ALTER TABLE `bar` ADD CONSTRAINT UNIQUE INDEX (`u2`)", "ALTER TABLE `bar` ADD CONSTRAINT UNIQUE INDEX (`u3`)"}, false}, - {"alter table bar add index (`name`), add index `hash_index` using hash (`name1`) COMMENT 'a hash index'", []string{"ALTER TABLE `bar` ADD INDEX (`name`)", "ALTER TABLE `bar` ADD INDEX `hash_index` USING HASH (`name1`) COMMENT 'a hash index'"}, false}, - {"alter table bar add index using btree (`name`), add unique index using hash (`age`), add primary key using btree (`id`)", []string{"ALTER TABLE `bar` ADD INDEX USING BTREE (`name`)", "ALTER TABLE `bar` ADD CONSTRAINT UNIQUE INDEX USING HASH (`age`)", "ALTER TABLE `bar` ADD CONSTRAINT PRIMARY KEY USING BTREE (`id`)"}, false}, - {"alter table bar add index (`name`), add CONSTRAINT `pp` FOREIGN KEY (product_category, product_id) REFERENCES product(category, id)", []string{"ALTER TABLE `bar` ADD INDEX (`name`)", "ALTER TABLE `bar` ADD CONSTRAINT `pp` FOREIGN KEY (`product_category`, `product_id`) REFERENCES `product` (`category`, `id`)"}, false}, - // According to tidb parser/parser.y: index order is parsed but just ignored as MySQL did. - {"alter tabLE bar ADD FULLTEXT INDEX `fulltext` (`name` ASC), add index (`c1`)", []string{"ALTER TABLE `bar` ADD FULLTEXT INDEX `fulltext` (`name`)", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - {"alter tabLE bar ADD FULLTEXT KEY `fulltext` (`name` ASC), add index (`c1`)", []string{"ALTER TABLE `bar` ADD FULLTEXT INDEX `fulltext` (`name`)", "ALTER TABLE `bar` ADD INDEX (`c1`)"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableDropForeignKey(c *C) { - tests := []testCase{ - {"alter table bar drop key a, drop FOREIGN KEY b", []string{"ALTER TABLE `bar` DROP INDEX `a`", "ALTER TABLE `bar` DROP FOREIGN KEY `b`"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableModifyColumn(c *C) { - tests := []testCase{ - {"alter table bar modify a varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci", []string{"ALTER TABLE `bar` MODIFY COLUMN `a` varchar(255)"}, false}, - {"alter table bar modify a varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci, modify b char(255) first, modify c text after d", []string{"ALTER TABLE `bar` MODIFY COLUMN `a` varchar(255)", "ALTER TABLE `bar` MODIFY COLUMN `b` char(255) FIRST", "ALTER TABLE `bar` MODIFY COLUMN `c` text AFTER `d`"}, false}, - {"alter table bar modify a enum('signup','unique','sliding') CHARACTER SET utf8 COLLATE utf8_general_ci, modify sites set('mt') CHARACTER SET utf8 COLLATE utf8_general_ci first, modify c text after d", []string{"ALTER TABLE `bar` MODIFY COLUMN `a` enum('signup','unique','sliding')", "ALTER TABLE `bar` MODIFY COLUMN `sites` set('mt') FIRST", "ALTER TABLE `bar` MODIFY COLUMN `c` text AFTER `d`"}, false}, - {"alter table bar modify c1 timestamp default now() ON UPDATE now(), drop c1", []string{"ALTER TABLE `bar` MODIFY COLUMN `c1` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP", "ALTER TABLE `bar` DROP COLUMN `c1`"}, false}, - {"alter table `bar` modify column `id` int(11) unsigned zerofill not null", []string{"ALTER TABLE `bar` MODIFY COLUMN `id` int(11) UNSIGNED ZEROFILL NOT NULL"}, false}, - {"alter table `page_question` modify column `pageId` bigint not null default ''", []string{"ALTER TABLE `page_question` MODIFY COLUMN `pageId` bigint(20) NOT NULL DEFAULT ''"}, false}, - {"alter table `page_question` modify column `pageId` binary not null default ''", []string{"ALTER TABLE `page_question` MODIFY COLUMN `pageId` binary(1) NOT NULL DEFAULT ''"}, false}, - {"alter table `page_question` modify column `pageId` varbinary(20) not null default ''", []string{"ALTER TABLE `page_question` MODIFY COLUMN `pageId` varbinary(20) NOT NULL DEFAULT ''"}, false}, - {"alter table `bar` modify column `id` varchar(20) BINARY not null default ''", []string{"ALTER TABLE `bar` MODIFY COLUMN `id` varchar(20) BINARY NOT NULL DEFAULT ''"}, false}, - {"alter table `bar` modify column `id` char(20) binary not null default ''", []string{"ALTER TABLE `bar` MODIFY COLUMN `id` char(20) BINARY NOT NULL DEFAULT ''"}, false}, - {"alter table bar modify column `id` tinyblob", []string{"ALTER TABLE `bar` MODIFY COLUMN `id` tinyblob"}, false}, - {"alter table bar modify column `id` blob", []string{"ALTER TABLE `bar` MODIFY COLUMN `id` blob"}, false}, - {"alter table bar modify column `id` mediumblob", []string{"ALTER TABLE `bar` MODIFY COLUMN `id` mediumblob"}, false}, - {"alter table bar modify column `id` longblob", []string{"ALTER TABLE `bar` MODIFY COLUMN `id` longblob"}, false}, - {"alter table bar modify column `id` text binary", []string{"ALTER TABLE `bar` MODIFY COLUMN `id` text BINARY"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableChangeColumn(c *C) { - tests := []testCase{ - {"alter table bar change a b text CHARACTER SET utf8 COLLATE utf8_general_ci, change c d tinytext CHARACTER SET utf8 COLLATE utf8_general_ci,change e f mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci, change g h longtext CHARACTER SET utf8 COLLATE utf8_general_ci", []string{"ALTER TABLE `bar` CHANGE COLUMN `a` `b` text", "ALTER TABLE `bar` CHANGE COLUMN `c` `d` tinytext", "ALTER TABLE `bar` CHANGE COLUMN `e` `f` mediumtext", "ALTER TABLE `bar` CHANGE COLUMN `g` `h` longtext"}, false}, - {"alter table bar change a b varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci null default null, change c d char(255) CHARACTER SET utf8 COLLATE utf8_general_ci", []string{"ALTER TABLE `bar` CHANGE COLUMN `a` `b` varchar(255) NULL DEFAULT NULL", "ALTER TABLE `bar` CHANGE COLUMN `c` `d` char(255)"}, false}, - {"alter table bar change program program enum('signup','unique','sliding') CHARACTER SET utf8 COLLATE utf8_general_ci not null, change sites sites set('mt') CHARACTER SET utf8 COLLATE utf8_general_ci", []string{"ALTER TABLE `bar` CHANGE COLUMN `program` `program` enum('signup','unique','sliding') NOT NULL", "ALTER TABLE `bar` CHANGE COLUMN `sites` `sites` set('mt')"}, false}, - {"alter table bar change a b varchar(255), change c d varchar(255) first, change e f varchar(255) after g", []string{"ALTER TABLE `bar` CHANGE COLUMN `a` `b` varchar(255)", "ALTER TABLE `bar` CHANGE COLUMN `c` `d` varchar(255) FIRST", "ALTER TABLE `bar` CHANGE COLUMN `e` `f` varchar(255) AFTER `g`"}, false}, - {"alter table `bar` change column `id` `id` int(11) unsigned zerofill not null", []string{"ALTER TABLE `bar` CHANGE COLUMN `id` `id` int(11) UNSIGNED ZEROFILL NOT NULL"}, false}, - {"alter table `page_question` change column `pageId` `pageId` bigint not null default ''", []string{"ALTER TABLE `page_question` CHANGE COLUMN `pageId` `pageId` bigint(20) NOT NULL DEFAULT ''"}, false}, - {"alter table `page_question` change column `pageId` `pageId` binary not null default ''", []string{"ALTER TABLE `page_question` CHANGE COLUMN `pageId` `pageId` binary(1) NOT NULL DEFAULT ''"}, false}, - {"alter table `page_question` change column `pageId` `pageId` varbinary(20) not null default ''", []string{"ALTER TABLE `page_question` CHANGE COLUMN `pageId` `pageId` varbinary(20) NOT NULL DEFAULT ''"}, false}, - {"alter table `bar` change column `id` `id` varchar(20) BINARY not null default ''", []string{"ALTER TABLE `bar` CHANGE COLUMN `id` `id` varchar(20) BINARY NOT NULL DEFAULT ''"}, false}, - {"alter table `bar` change column `id` `id` char(20) binary not null default ''", []string{"ALTER TABLE `bar` CHANGE COLUMN `id` `id` char(20) BINARY NOT NULL DEFAULT ''"}, false}, - {"alter table bar change column `id` `id` tinyblob", []string{"ALTER TABLE `bar` CHANGE COLUMN `id` `id` tinyblob"}, false}, - {"alter table bar change column `id` `id` blob", []string{"ALTER TABLE `bar` CHANGE COLUMN `id` `id` blob"}, false}, - {"alter table bar change column `id` `id` mediumblob", []string{"ALTER TABLE `bar` CHANGE COLUMN `id` `id` mediumblob"}, false}, - {"alter table bar change column `id` `id` longblob", []string{"ALTER TABLE `bar` CHANGE COLUMN `id` `id` longblob"}, false}, - {"alter table bar change column `id` `id` text binary", []string{"ALTER TABLE `bar` CHANGE COLUMN `id` `id` text BINARY"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableRenameTable(c *C) { - tests := []testCase{ - {"alter table bar add index (id), rename to bar1", []string{"ALTER TABLE `bar` ADD INDEX (`id`)", "ALTER TABLE `bar` RENAME TO `bar1`"}, false}, - {"alter table bar rename to bar1, add index (id)", []string{"ALTER TABLE `bar` RENAME TO `bar1`", "ALTER TABLE `bar1` ADD INDEX (`id`)"}, false}, - {"alter table bar rename to bar1, rename to bar2", []string{"ALTER TABLE `bar` RENAME TO `bar1`", "ALTER TABLE `bar1` RENAME TO `bar2`"}, false}, - {"alter table bar add index (id), rename as bar1, drop index id", []string{"ALTER TABLE `bar` ADD INDEX (`id`)", "ALTER TABLE `bar` RENAME TO `bar1`", "ALTER TABLE `bar1` DROP INDEX `id`"}, false}, - {"alter table foo.bar rename to foo.bar1, add index (id)", []string{"ALTER TABLE `foo`.`bar` RENAME TO `foo`.`bar1`", "ALTER TABLE `foo`.`bar1` ADD INDEX (`id`)"}, false}, - {"alter table foo.bar add index (id), rename as bar1", []string{"ALTER TABLE `foo`.`bar` ADD INDEX (`id`)", "ALTER TABLE `foo`.`bar` RENAME TO `bar1`"}, false}, - {"alter table bar1 add index (cat1), add index (cat2), rename to bar", []string{"ALTER TABLE `bar1` ADD INDEX (`cat1`)", "ALTER TABLE `bar1` ADD INDEX (`cat2`)", "ALTER TABLE `bar1` RENAME TO `bar`"}, false}, - {"rename table `t1` to `t2`, `t3` to `t4`", []string{"RENAME TABLE `t1` TO `t2`", "RENAME TABLE `t3` TO `t4`"}, false}, - {"rename table `db`.`t1` to `db`.`t2`, `db`.`t3` to `db`.`t4`", []string{"RENAME TABLE `db`.`t1` TO `db`.`t2`", "RENAME TABLE `db`.`t3` TO `db`.`t4`"}, false}, - {"alter table bar rename index idx_1 to idx_2, rename key idx_3 to idx_4", []string{"ALTER TABLE `bar` RENAME INDEX `idx_1` TO `idx_2`", "ALTER TABLE `bar` RENAME INDEX `idx_3` TO `idx_4`"}, false}, - {"alter table bar rename index `idx_1` to `idx_2`, rename key `idx_3` to `idx_4`", []string{"ALTER TABLE `bar` RENAME INDEX `idx_1` TO `idx_2`", "ALTER TABLE `bar` RENAME INDEX `idx_3` TO `idx_4`"}, false}, - {"alter table bar rename index idx_1 to idx_2", []string{"ALTER TABLE `bar` RENAME INDEX `idx_1` TO `idx_2`"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableAlterColumn(c *C) { - tests := []testCase{ - {"alter table bar alter `id` set default 1, alter `name` drop default", []string{"ALTER TABLE `bar` ALTER COLUMN `id` SET DEFAULT 1", "ALTER TABLE `bar` ALTER COLUMN `name` DROP DEFAULT"}, false}, - {"alter table bar alter column `ctime` set default '2018-01-01 01:01:01'", []string{"ALTER TABLE `bar` ALTER COLUMN `ctime` SET DEFAULT '2018-01-01 01:01:01'"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableDropPrimaryKey(c *C) { - tests := []testCase{ - {"alter table bar DROP PRIMARY KEY, drop a", []string{"ALTER TABLE `bar` DROP PRIMARY KEY", "ALTER TABLE `bar` DROP COLUMN `a`"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableLock(c *C) { - tests := []testCase{ - {"alter table `foo`.`bar` add index `idx_t` (`create_time`), lock=none", []string{"ALTER TABLE `foo`.`bar` ADD INDEX `idx_t` (`create_time`)"}, false}, - {"alter table `foo`.`bar` add index `idx_t` (`create_time`), lock=default", []string{"ALTER TABLE `foo`.`bar` ADD INDEX `idx_t` (`create_time`)"}, false}, - {"alter table `foo`.`bar` add index `idx_t` (`create_time`), lock=shared", []string{"ALTER TABLE `foo`.`bar` ADD INDEX `idx_t` (`create_time`)"}, false}, - {"alter table `foo`.`bar` add index `idx_t` (`create_time`), lock=exclusive", []string{"ALTER TABLE `foo`.`bar` ADD INDEX `idx_t` (`create_time`)"}, false}, - } - - s.run(c, tests) -} - -func (s *testSyncerSuite) testAlterTableConvert(c *C) { - tests := []testCase{ - {"alter table `bar` CONVERT TO CHARACTER SET utf8", []string{"ALTER TABLE `bar` CHARACTER SET = utf8"}, false}, - {"alter table `bar` CONVERT TO CHARSET utf8", []string{"ALTER TABLE `bar` CHARACTER SET = utf8"}, false}, - {"alter table `bar` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin", []string{"ALTER TABLE `bar` CHARACTER SET = utf8 COLLATE = utf8_bin"}, false}, - {"alter table `bar` CONVERT TO CHARSET utf8 COLLATE utf8_bin", []string{"ALTER TABLE `bar` CHARACTER SET = utf8 COLLATE = utf8_bin"}, false}, - {"ALTER TABLE `foo`.`bar` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin", []string{"ALTER TABLE `foo`.`bar` CHARACTER SET = utf8 COLLATE = utf8_bin"}, false}, - } - s.run(c, tests) -} - -func (s *testSyncerSuite) run(c *C, tests []testCase) { - parser, err := utils.GetParser(s.db, false) - c.Assert(err, IsNil) - - syncer := &Syncer{} - - for _, tt := range tests { - sqls, _, _, err := syncer.resolveDDLSQL(tt.sql, parser, "") - if !tt.wantErr && err != nil { - fmt.Println(err) - } - if tt.wantErr { - c.Assert(err, NotNil) - } else { - c.Assert(err, IsNil) - } - c.Assert(sqls, DeepEquals, tt.wantSqls) - } -} diff --git a/syncer/ddl.go b/syncer/ddl.go index e02c31f716..a7fc69f297 100644 --- a/syncer/ddl.go +++ b/syncer/ddl.go @@ -14,8 +14,6 @@ package syncer import ( - "fmt" - ddlpkg "github.com/pingcap/dm/pkg/ddl" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/errors" @@ -128,54 +126,17 @@ func (s *Syncer) parseDDLSQL(sql string, p *parser.Parser, schema string) (resul } } -// resolveDDLSQL resolve to one ddl sql -// example: drop table test.a,test2.b -> drop table test.a; drop table test2.b; -func (s *Syncer) resolveDDLSQL(sql string, p *parser.Parser, schema string) (sqls []string, tables map[string]*filter.Table, isDDL bool, err error) { - // would remove it later - parseResult, err := s.parseDDLSQL(sql, p, schema) +/// resolveDDLSQL do two things +// * it splits multiple operations in one DDL statement into multiple DDL statements +// * try to apply online ddl by given online +// return @spilted sqls, @online ddl table names, @error +func (s *Syncer) resolveDDLSQL(p *parser.Parser, stmt ast.StmtNode, schema string) (sqls []string, tables map[string]*filter.Table, err error) { + sqls, err = ddlpkg.SplitDDL(stmt, schema) if err != nil { - return []string{sql}, nil, false, errors.Trace(err) - } - if !parseResult.isDDL { - return nil, nil, false, nil - } - - switch v := parseResult.stmt.(type) { - case *ast.DropTableStmt: - var ex string - if v.IfExists { - ex = "IF EXISTS " - } - for _, t := range v.Tables { - var db string - if t.Schema.O != "" { - db = fmt.Sprintf("`%s`.", t.Schema.O) - } - s := fmt.Sprintf("DROP TABLE %s%s`%s`", ex, db, t.Name.O) - sqls = append(sqls, s) - } - case *ast.AlterTableStmt: - tempSpecs := v.Specs - newTable := &ast.TableName{} - log.Warnf("will split alter table statement: %v", sql) - for i := range tempSpecs { - v.Specs = tempSpecs[i : i+1] - splitted := alterTableStmtToSQL(v, newTable) - log.Warnf("splitted alter table statement: %v", splitted) - sqls = append(sqls, splitted...) - } - case *ast.RenameTableStmt: - for _, t2t := range v.TableToTables { - sqlNew := fmt.Sprintf("RENAME TABLE %s TO %s", tableNameToSQL(t2t.OldTable), tableNameToSQL(t2t.NewTable)) - sqls = append(sqls, sqlNew) - } - - default: - sqls = append(sqls, sql) + return nil, nil, errors.Trace(err) } - if s.onlineDDL == nil { - return sqls, nil, true, nil + return sqls, nil, nil } statements := make([]string, 0, len(sqls)) @@ -184,7 +145,7 @@ func (s *Syncer) resolveDDLSQL(sql string, p *parser.Parser, schema string) (sql // filter and store ghost table ddl, transform online ddl ss, tableName, err := s.handleOnlineDDL(p, schema, sql) if err != nil { - return statements, tables, true, errors.Trace(err) + return statements, tables, errors.Trace(err) } if tableName != nil { @@ -193,180 +154,7 @@ func (s *Syncer) resolveDDLSQL(sql string, p *parser.Parser, schema string) (sql statements = append(statements, ss...) } - return statements, tables, true, nil -} - -// todo: fix the ugly code, use ast to rename table -func genDDLSQL(sql string, stmt ast.StmtNode, originTableNames []*filter.Table, targetTableNames []*filter.Table, addUseDatabasePrefix bool) (string, error) { - addUseDatabase := func(sql string, dbName string) string { - if addUseDatabasePrefix { - return fmt.Sprintf("USE `%s`; %s;", dbName, sql) - } - - return sql - } - - if notNeedRoute(originTableNames, targetTableNames) { - _, isCreateDatabase := stmt.(*ast.CreateDatabaseStmt) - if isCreateDatabase { - return fmt.Sprintf("%s;", sql), nil - } - - return addUseDatabase(sql, originTableNames[0].Schema), nil - } - - switch stmt.(type) { - case *ast.CreateDatabaseStmt: - sqlPrefix := createDatabaseRegex.FindString(sql) - index := findLastWord(sqlPrefix) - return createDatabaseRegex.ReplaceAllString(sql, fmt.Sprintf("%s`%s`", sqlPrefix[:index], targetTableNames[0].Schema)), nil - - case *ast.DropDatabaseStmt: - sqlPrefix := dropDatabaseRegex.FindString(sql) - index := findLastWord(sqlPrefix) - return dropDatabaseRegex.ReplaceAllString(sql, fmt.Sprintf("%s`%s`", sqlPrefix[:index], targetTableNames[0].Schema)), nil - - case *ast.CreateTableStmt: - var ( - sqlPrefix string - index int - ) - // replace `like schema.table` section - if len(originTableNames) == 2 { - sqlPrefix = createTableLikeRegex.FindString(sql) - index = findLastWord(sqlPrefix) - endChars := "" - if sqlPrefix[len(sqlPrefix)-1] == ')' { - endChars = ")" - } - sql = createTableLikeRegex.ReplaceAllString(sql, fmt.Sprintf("%s`%s`.`%s`%s", sqlPrefix[:index], targetTableNames[1].Schema, targetTableNames[1].Name, endChars)) - } - // replce `create table schame.table` section - sqlPrefix = createTableRegex.FindString(sql) - index = findLastWord(sqlPrefix) - endChars := findTableDefineIndex(sqlPrefix[index:]) - sql = createTableRegex.ReplaceAllString(sql, fmt.Sprintf("%s`%s`.`%s`%s", sqlPrefix[:index], targetTableNames[0].Schema, targetTableNames[0].Name, endChars)) - - case *ast.DropTableStmt: - sqlPrefix := dropTableRegex.FindString(sql) - index := findLastWord(sqlPrefix) - sql = dropTableRegex.ReplaceAllString(sql, fmt.Sprintf("%s`%s`.`%s`", sqlPrefix[:index], targetTableNames[0].Schema, targetTableNames[0].Name)) - - case *ast.TruncateTableStmt: - sql = fmt.Sprintf("TRUNCATE TABLE `%s`.`%s`", targetTableNames[0].Schema, targetTableNames[0].Name) - - case *ast.AlterTableStmt: - // RENAME [TO|AS] new_tbl_name - if len(originTableNames) == 2 { - index := findLastWord(sql) - sql = fmt.Sprintf("%s`%s`.`%s`", sql[:index], targetTableNames[1].Schema, targetTableNames[1].Name) - } - sql = alterTableRegex.ReplaceAllString(sql, fmt.Sprintf("ALTER TABLE `%s`.`%s`", targetTableNames[0].Schema, targetTableNames[0].Name)) - - case *ast.RenameTableStmt: - return fmt.Sprintf("RENAME TABLE `%s`.`%s` TO `%s`.`%s`", targetTableNames[0].Schema, targetTableNames[0].Name, - targetTableNames[1].Schema, targetTableNames[1].Name), nil - - case *ast.CreateIndexStmt: - sql = createIndexDDLRegex.ReplaceAllString(sql, fmt.Sprintf("ON `%s`.`%s` (", targetTableNames[0].Schema, targetTableNames[0].Name)) - - case *ast.DropIndexStmt: - sql = dropIndexDDLRegex.ReplaceAllString(sql, fmt.Sprintf("ON `%s`.`%s`", targetTableNames[0].Schema, targetTableNames[0].Name)) - - default: - return "", errors.Errorf("unkown type ddl %s", sql) - } - - return addUseDatabase(sql, targetTableNames[0].Schema), nil -} - -func notNeedRoute(originTableNames []*filter.Table, targetTableNames []*filter.Table) bool { - for index, originTableName := range originTableNames { - targetTableName := targetTableNames[index] - if originTableName.Schema != targetTableName.Schema { - return false - } - if originTableName.Name != targetTableName.Name { - return false - } - } - return true -} - -func findLastWord(literal string) int { - index := len(literal) - 1 - for index >= 0 && literal[index] == ' ' { - index-- - } - - for index >= 0 { - if literal[index-1] == ' ' { - return index - } - index-- - } - return index -} - -func findTableDefineIndex(literal string) string { - for i := range literal { - if literal[i] == '(' { - return literal[i:] - } - } - return "" -} - -func genTableName(schema string, table string) *filter.Table { - return &filter.Table{Schema: schema, Name: table} - -} - -// the result contains [tableName] excepted create table like and rename table -// for `create table like` DDL, result contains [sourceTableName, sourceRefTableName] -// for rename table ddl, result contains [targetOldTableName, sourceNewTableName] -func fetchDDLTableNames(schema string, stmt ast.StmtNode) ([]*filter.Table, error) { - var res []*filter.Table - switch v := stmt.(type) { - case *ast.CreateDatabaseStmt: - res = append(res, genTableName(v.Name, "")) - case *ast.DropDatabaseStmt: - res = append(res, genTableName(v.Name, "")) - case *ast.CreateTableStmt: - res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) - if v.ReferTable != nil { - res = append(res, genTableName(v.ReferTable.Schema.O, v.ReferTable.Name.O)) - } - case *ast.DropTableStmt: - if len(v.Tables) != 1 { - return res, errors.Errorf("drop table with multiple tables, may resovle ddl sql failed") - } - res = append(res, genTableName(v.Tables[0].Schema.O, v.Tables[0].Name.O)) - case *ast.TruncateTableStmt: - res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) - case *ast.AlterTableStmt: - res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) - if v.Specs[0].NewTable != nil { - res = append(res, genTableName(v.Specs[0].NewTable.Schema.O, v.Specs[0].NewTable.Name.O)) - } - case *ast.RenameTableStmt: - res = append(res, genTableName(v.OldTable.Schema.O, v.OldTable.Name.O)) - res = append(res, genTableName(v.NewTable.Schema.O, v.NewTable.Name.O)) - case *ast.CreateIndexStmt: - res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) - case *ast.DropIndexStmt: - res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) - default: - return res, errors.Errorf("unkown type ddl %s", stmt) - } - - for i := range res { - if res[i].Schema == "" { - res[i].Schema = schema - } - } - - return res, nil + return statements, tables, nil } func (s *Syncer) handleDDL(p *parser.Parser, schema, sql string) (string, [][]*filter.Table, ast.StmtNode, error) { @@ -375,7 +163,7 @@ func (s *Syncer) handleDDL(p *parser.Parser, schema, sql string) (string, [][]*f return "", nil, nil, errors.Annotatef(err, "ddl %s", sql) } - tableNames, err := fetchDDLTableNames(schema, stmt) + tableNames, err := ddlpkg.FetchDDLTableNames(schema, stmt) if err != nil { return "", nil, nil, errors.Trace(err) } @@ -398,7 +186,7 @@ func (s *Syncer) handleDDL(p *parser.Parser, schema, sql string) (string, [][]*f targetTableNames = append(targetTableNames, tableName) } - ddl, err := genDDLSQL(sql, stmt, tableNames, targetTableNames, true) + ddl, err := ddlpkg.RenameDDLTable(stmt, targetTableNames) return ddl, [][]*filter.Table{tableNames, targetTableNames}, stmt, errors.Trace(err) } @@ -414,7 +202,7 @@ func (s *Syncer) handleOnlineDDL(p *parser.Parser, schema, sql string) ([]string return nil, nil, errors.Annotatef(err, "ddl %s", sql) } - tableNames, err := fetchDDLTableNames(schema, stmt) + tableNames, err := ddlpkg.FetchDDLTableNames(schema, stmt) if err != nil { return nil, nil, errors.Trace(err) } @@ -439,7 +227,7 @@ func (s *Syncer) handleOnlineDDL(p *parser.Parser, schema, sql string) ([]string return nil, nil, errors.Trace(err) } - sqls[i], err = genDDLSQL(sqls[i], stmt, tableNames[:1], targetTables, false) + sqls[i], err = ddlpkg.RenameDDLTable(stmt, targetTables) if err != nil { return nil, nil, errors.Trace(err) } diff --git a/syncer/inject_sql.go b/syncer/inject_sql.go index 9afe639284..3268d6fb4b 100644 --- a/syncer/inject_sql.go +++ b/syncer/inject_sql.go @@ -16,6 +16,7 @@ package syncer import ( "time" + ddlpkg "github.com/pingcap/dm/pkg/ddl" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/errors" "github.com/pingcap/parser" @@ -41,7 +42,7 @@ func (s *Syncer) InjectSQLs(ctx context.Context, sqls []string) error { if !ok { return errors.Errorf("only support inject DDL for sharding group to be synced currently, but got %s", sql) } - tableNames, err := fetchDDLTableNames("", ddlNode) + tableNames, err := ddlpkg.FetchDDLTableNames("", ddlNode) if err != nil { return errors.Trace(err) } diff --git a/syncer/online_ddl.go b/syncer/online_ddl.go index cc1e5fee0a..318f0d695c 100644 --- a/syncer/online_ddl.go +++ b/syncer/online_ddl.go @@ -17,6 +17,7 @@ import ( "encoding/json" "fmt" "strconv" + "strings" "sync" "github.com/pingcap/errors" @@ -284,3 +285,7 @@ func (s *OnlineDDLStorage) createTable() error { err := s.db.executeSQL([]string{sql}, [][]interface{}{nil}, maxRetryCount) return errors.Trace(err) } + +func escapeSingleQuote(str string) string { + return strings.Replace(str, "'", "''", -1) +} diff --git a/syncer/syncer.go b/syncer/syncer.go index 3b9e4d1ece..cf5bd7a4f5 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -1283,7 +1283,7 @@ func (s *Syncer) Run(ctx context.Context) (err error) { // for DDL, we don't apply operator until we try to execute it. // so can handle sharding cases - sqls, onlineDDLTableNames, _, err = s.resolveDDLSQL(sql, parser2, string(ev.Schema)) + sqls, onlineDDLTableNames, err = s.resolveDDLSQL(parser2, parseResult.stmt, string(ev.Schema)) if err != nil { log.Infof("[query]%s [last pos]%v [current pos]%v [current gtid set]%v", sql, lastPos, currentPos, ev.GSet) log.Errorf("fail to be parsed, error %v", err) From cbd79667a648a7ff26c55f2651ec59b36b485761 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Tue, 19 Feb 2019 18:28:09 +0800 Subject: [PATCH 03/20] *: remove usless code --- syncer/filter.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/syncer/filter.go b/syncer/filter.go index ac9a7023a9..36c6bf95b5 100644 --- a/syncer/filter.go +++ b/syncer/filter.go @@ -14,7 +14,6 @@ package syncer import ( - "fmt" "regexp" "strings" @@ -30,23 +29,6 @@ CREATE [TEMPORARY] TABLE [IF NOT EXISTS] tbl_name { LIKE old_tbl_name | (LIKE old_tbl_name) } */ var ( - commentRegexStr = "(\\s*(/\\*.*\\*/)*\\s*)*" - // https://dev.mysql.com/doc/refman/5.7/en/create-database.html - createDatabaseRegex = regexp.MustCompile(fmt.Sprintf("(?i)CREATE%s\\s+(DATABASE|SCHEMA)\\s+(IF NOT EXISTS\\s+)?\\S+", commentRegexStr)) - // https://dev.mysql.com/doc/refman/5.7/en/drop-database.html - dropDatabaseRegex = regexp.MustCompile(fmt.Sprintf("(?i)DROP%s\\s+(DATABASE|SCHEMA)\\s+(IF EXISTS\\s+)?\\S+", commentRegexStr)) - // https://dev.mysql.com/doc/refman/5.7/en/create-index.html - // https://dev.mysql.com/doc/refman/5.7/en/drop-index.html - createIndexDDLRegex = regexp.MustCompile("(?i)ON\\s+\\S+\\s*\\(") - dropIndexDDLRegex = regexp.MustCompile("(?i)ON\\s+\\S+") - // https://dev.mysql.com/doc/refman/5.7/en/create-table.html - createTableRegex = regexp.MustCompile(fmt.Sprintf("(?i)CREATE%s\\s+(TEMPORARY\\s+)?TABLE\\s+(IF NOT EXISTS\\s+)?\\S+", commentRegexStr)) - createTableLikeRegex = regexp.MustCompile(fmt.Sprintf("(?i)CREATE%s\\s+(TEMPORARY\\s+)?TABLE\\s+(IF NOT EXISTS\\s+)?\\S+\\s*\\(?\\s*LIKE\\s+\\S+", commentRegexStr)) - // https://dev.mysql.com/doc/refman/5.7/en/drop-table.html - dropTableRegex = regexp.MustCompile(fmt.Sprintf("^(?i)DROP%s\\s+(TEMPORARY\\s+)?TABLE\\s+(IF EXISTS\\s+)?\\S+", commentRegexStr)) - // https://dev.mysql.com/doc/refman/5.7/en/alter-table.html - alterTableRegex = regexp.MustCompile(fmt.Sprintf("^(?i)ALTER%s\\s+TABLE\\s+\\S+", commentRegexStr)) - // https://dev.mysql.com/doc/refman/5.7/en/create-trigger.html builtInSkipDDLs = []string{ // For mariadb, for query event, like `# Dumm` // But i don't know what is the meaning of this event. From fb5c27851434e3e4344e667539a5d7867324af69 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Wed, 20 Feb 2019 18:40:58 +0800 Subject: [PATCH 04/20] *: refine code --- dm/ctl/common/util.go | 4 +- loader/convert_data.go | 4 +- pkg/parser/commen_test.go | 212 ++++++++++++++++++++++++++++++++++ pkg/{ddl => parser}/common.go | 32 ++++- relay/util.go | 4 +- syncer/ddl.go | 14 +-- syncer/ddl_test.go | 156 ++----------------------- syncer/inject_sql.go | 4 +- syncer/syncer_test.go | 13 ++- 9 files changed, 267 insertions(+), 176 deletions(-) create mode 100644 pkg/parser/commen_test.go rename pkg/{ddl => parser}/common.go (91%) diff --git a/dm/ctl/common/util.go b/dm/ctl/common/util.go index ce25d9ad76..440e906c27 100644 --- a/dm/ctl/common/util.go +++ b/dm/ctl/common/util.go @@ -22,7 +22,7 @@ import ( "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" "github.com/pingcap/dm/dm/pb" - "github.com/pingcap/dm/pkg/ddl" + parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -166,7 +166,7 @@ func ExtractSQLsFromArgs(args []string) ([]string, error) { concat := strings.TrimSpace(strings.Join(args, " ")) parser := parser.New() - nodes, err := ddl.Parse(parser, concat, "", "") + nodes, err := parserpkg.Parse(parser, concat, "", "") if err != nil { return nil, errors.Annotatef(err, "invalid sql '%s'", concat) } diff --git a/loader/convert_data.go b/loader/convert_data.go index 54acf674e1..f74b7b502a 100644 --- a/loader/convert_data.go +++ b/loader/convert_data.go @@ -22,7 +22,7 @@ import ( "strings" "unsafe" - "github.com/pingcap/dm/pkg/ddl" + parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -218,7 +218,7 @@ func parseTable(r *router.Table, schema, table, file string) (*tableInfo, error) return nil, errors.Annotatef(err, "read table info from file %s", file) } - stmts, err := ddl.Parse(parser.New(), string(statement), "", "") + stmts, err := parserpkg.Parse(parser.New(), string(statement), "", "") if err != nil { return nil, errors.Annotatef(err, "parser statement %s", statement) } diff --git a/pkg/parser/commen_test.go b/pkg/parser/commen_test.go new file mode 100644 index 0000000000..8ece6b0ec9 --- /dev/null +++ b/pkg/parser/commen_test.go @@ -0,0 +1,212 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "testing" + + . "github.com/pingcap/check" + "github.com/pingcap/parser" + "github.com/pingcap/tidb-tools/pkg/filter" +) + +var _ = Suite(&testParserSuite{}) + +var sqls = []string{ + "create schema `s1`", + "create schema if not exists `s1`", + "drop schema `s1`", + "drop schema if exists `s1`", + "drop table `s1`.`t1`", + "drop table `s1`.`t1`, `s2`.`t2`", + "drop table `s1`.`t1`, `s2`.`t2`, `xx`", + "create table `s1`.`t1` (id int)", + "create table `t1` (id int)", + "create table `t1` like `t2`", + "create table `s1`.`t1` like `t2`", + "create table `t1` like `xx`.`t2`", + "truncate table `t1`", + "truncate table `s1`.`t1`", + "rename table `s1`.`t1` to `s2`.`t2`", + "rename table `t1` to `t2`, `s1`.`t1` to `t2`", + "drop index i1 on `s1`.`t1`", + "drop index i1 on `t1`", + "create index i1 on `t1`(`c1`)", + "create index i1 on `s1`.`t1`(`c1`)", + "alter table `t1` add column c1 int, drop column c2", + "alter table `s1`.`t1` add column c1 int, rename to `t2`, drop column c2", + "alter table `s1`.`t1` add column c1 int, rename to `xx`.`t2`, drop column c2", +} + +func TestSuite(t *testing.T) { + TestingT(t) +} + +type testParserSuite struct { +} + +func (t *testParserSuite) TestParser(c *C) { + p := parser.New() + + for _, sql := range sqls { + stmts, err := Parse(p, sql, "", "") + c.Assert(err, IsNil) + c.Assert(stmts, HasLen, 1) + } + + unsupportedSQLs := []string{ + "alter table bar ADD FULLTEXT INDEX `fulltext` (`name`) WITH PARSER ngram", + "alter table bar ADD SPATIAL INDEX (`g`)", + "alter table bar ORDER BY id1, id2", + "alter table bar add index (`name`), add FOREIGN KEY (product_category, product_id) REFERENCES product(category, id) ON UPDATE CASCADE ON DELETE RESTRICT", + } + + for _, sql := range unsupportedSQLs { + _, err := Parse(p, sql, "", "") + c.Assert(err, NotNil) + } +} + +func (t *testParserSuite) TestResolveDDL(c *C) { + p := parser.New() + expectedSQLs := [][]string{ + {"CREATE DATABASE IF NOT EXISTS `s1`"}, + {"CREATE DATABASE IF NOT EXISTS `s1`"}, + {"DROP DATABASE IF EXISTS `s1`"}, + {"DROP DATABASE IF EXISTS `s1`"}, + {"DROP TABLE IF EXISTS `s1`.`t1`"}, + {"DROP TABLE IF EXISTS `s1`.`t1`", "DROP TABLE IF EXISTS `s2`.`t2`"}, + {"DROP TABLE IF EXISTS `s1`.`t1`", "DROP TABLE IF EXISTS `s2`.`t2`", "DROP TABLE IF EXISTS `test`.`xx`"}, + {"CREATE TABLE IF NOT EXISTS `s1`.`t1` (`id` INT)"}, + {"CREATE TABLE IF NOT EXISTS `test`.`t1` (`id` INT)"}, + {"CREATE TABLE IF NOT EXISTS `test`.`t1` LIKE `test`.`t2`"}, + {"CREATE TABLE IF NOT EXISTS `s1`.`t1` LIKE `test`.`t2`"}, + {"CREATE TABLE IF NOT EXISTS `test`.`t1` LIKE `xx`.`t2`"}, + {"TRUNCATE TABLE `test`.`t1`"}, + {"TRUNCATE TABLE `s1`.`t1`"}, + {"RENAME TABLE `s1`.`t1` TO `s2`.`t2`"}, + {"RENAME TABLE `test`.`t1` TO `test`.`t2`", "RENAME TABLE `s1`.`t1` TO `test`.`t2`"}, + {"DROP INDEX IF EXISTS `i1` ON `s1`.`t1`"}, + {"DROP INDEX IF EXISTS `i1` ON `test`.`t1`"}, + {"CREATE INDEX `i1` ON `test`.`t1` (`c1`)"}, + {"CREATE INDEX `i1` ON `s1`.`t1` (`c1`)"}, + {"ALTER TABLE `test`.`t1` ADD COLUMN `c1` INT", "ALTER TABLE `test`.`t1` DROP COLUMN `c2`"}, + {"ALTER TABLE `s1`.`t1` ADD COLUMN `c1` INT", "ALTER TABLE `s1`.`t1` RENAME AS `test`.`t2`", "ALTER TABLE `test`.`t2` DROP COLUMN `c2`"}, + {"ALTER TABLE `s1`.`t1` ADD COLUMN `c1` INT", "ALTER TABLE `s1`.`t1` RENAME AS `xx`.`t2`", "ALTER TABLE `xx`.`t2` DROP COLUMN `c2`"}, + } + + expectedTableName := [][][]*filter.Table{ + {{genTableName("s1", "")}}, + {{genTableName("s1", "")}}, + {{genTableName("s1", "")}}, + {{genTableName("s1", "")}}, + {{genTableName("s1", "t1")}}, + {{genTableName("s1", "t1")}, {genTableName("s2", "t2")}}, + {{genTableName("s1", "t1")}, {genTableName("s2", "t2")}, {genTableName("test", "xx")}}, + {{genTableName("s1", "t1")}}, + {{genTableName("test", "t1")}}, + {{genTableName("test", "t1"), genTableName("test", "t2")}}, + {{genTableName("s1", "t1"), genTableName("test", "t2")}}, + {{genTableName("test", "t1"), genTableName("xx", "t2")}}, + {{genTableName("test", "t1")}}, + {{genTableName("s1", "t1")}}, + {{genTableName("s1", "t1"), genTableName("s2", "t2")}}, + {{genTableName("test", "t1"), genTableName("test", "t2")}, {genTableName("s1", "t1"), genTableName("test", "t2")}}, + {{genTableName("s1", "t1")}}, + {{genTableName("test", "t1")}}, + {{genTableName("test", "t1")}}, + {{genTableName("s1", "t1")}}, + {{genTableName("test", "t1")}, {genTableName("test", "t1")}}, + {{genTableName("s1", "t1")}, {genTableName("s1", "t1"), genTableName("test", "t2")}, {genTableName("test", "t2")}}, + {{genTableName("s1", "t1")}, {genTableName("s1", "t1"), genTableName("xx", "t2")}, {genTableName("xx", "t2")}}, + } + + targetTableNames := [][][]*filter.Table{ + {{genTableName("xs1", "")}}, + {{genTableName("xs1", "")}}, + {{genTableName("xs1", "")}}, + {{genTableName("xs1", "")}}, + {{genTableName("xs1", "xt1")}}, + {{genTableName("xs1", "xt1")}, {genTableName("xs2", "xt2")}}, + {{genTableName("xs1", "xt1")}, {genTableName("xs2", "xt2")}, {genTableName("xtest", "xxx")}}, + {{genTableName("xs1", "xt1")}}, + {{genTableName("xtest", "xt1")}}, + {{genTableName("xtest", "xt1"), genTableName("xtest", "xt2")}}, + {{genTableName("xs1", "xt1"), genTableName("xtest", "xt2")}}, + {{genTableName("xtest", "xt1"), genTableName("xxx", "xt2")}}, + {{genTableName("xtest", "xt1")}}, + {{genTableName("xs1", "xt1")}}, + {{genTableName("xs1", "xt1"), genTableName("xs2", "xt2")}}, + {{genTableName("xtest", "xt1"), genTableName("xtest", "xt2")}, {genTableName("xs1", "xt1"), genTableName("xtest", "xt2")}}, + {{genTableName("xs1", "xt1")}}, + {{genTableName("xtest", "xt1")}}, + {{genTableName("xtest", "xt1")}}, + {{genTableName("xs1", "xt1")}}, + {{genTableName("xtest", "xt1")}, {genTableName("xtest", "xt1")}}, + {{genTableName("xs1", "xt1")}, {genTableName("xs1", "xt1"), genTableName("xtest", "xt2")}, {genTableName("xtest", "xt2")}}, + {{genTableName("xs1", "xt1")}, {genTableName("xs1", "xt1"), genTableName("xxx", "xt2")}, {genTableName("xxx", "xt2")}}, + } + + targetSQLs := [][]string{ + {"CREATE DATABASE IF NOT EXISTS `xs1`"}, + {"CREATE DATABASE IF NOT EXISTS `xs1`"}, + {"DROP DATABASE IF EXISTS `xs1`"}, + {"DROP DATABASE IF EXISTS `xs1`"}, + {"DROP TABLE IF EXISTS `xs1`.`xt1`"}, + {"DROP TABLE IF EXISTS `xs1`.`xt1`", "DROP TABLE IF EXISTS `xs2`.`xt2`"}, + {"DROP TABLE IF EXISTS `xs1`.`xt1`", "DROP TABLE IF EXISTS `xs2`.`xt2`", "DROP TABLE IF EXISTS `xtest`.`xxx`"}, + {"CREATE TABLE IF NOT EXISTS `xs1`.`xt1` (`id` INT)"}, + {"CREATE TABLE IF NOT EXISTS `xtest`.`xt1` (`id` INT)"}, + {"CREATE TABLE IF NOT EXISTS `xtest`.`xt1` LIKE `xtest`.`xt2`"}, + {"CREATE TABLE IF NOT EXISTS `xs1`.`xt1` LIKE `xtest`.`xt2`"}, + {"CREATE TABLE IF NOT EXISTS `xtest`.`xt1` LIKE `xxx`.`xt2`"}, + {"TRUNCATE TABLE `xtest`.`xt1`"}, + {"TRUNCATE TABLE `xs1`.`xt1`"}, + {"RENAME TABLE `xs1`.`xt1` TO `xs2`.`xt2`"}, + {"RENAME TABLE `xtest`.`xt1` TO `xtest`.`xt2`", "RENAME TABLE `xs1`.`xt1` TO `xtest`.`xt2`"}, + {"DROP INDEX IF EXISTS `i1` ON `xs1`.`xt1`"}, + {"DROP INDEX IF EXISTS `i1` ON `xtest`.`xt1`"}, + {"CREATE INDEX `i1` ON `xtest`.`xt1` (`c1`)"}, + {"CREATE INDEX `i1` ON `xs1`.`xt1` (`c1`)"}, + {"ALTER TABLE `xtest`.`xt1` ADD COLUMN `c1` INT", "ALTER TABLE `xtest`.`xt1` DROP COLUMN `c2`"}, + {"ALTER TABLE `xs1`.`xt1` ADD COLUMN `c1` INT", "ALTER TABLE `xs1`.`xt1` RENAME AS `xtest`.`xt2`", "ALTER TABLE `xtest`.`xt2` DROP COLUMN `c2`"}, + {"ALTER TABLE `xs1`.`xt1` ADD COLUMN `c1` INT", "ALTER TABLE `xs1`.`xt1` RENAME AS `xxx`.`xt2`", "ALTER TABLE `xxx`.`xt2` DROP COLUMN `c2`"}, + } + + for i, sql := range sqls { + stmts, err := Parse(p, sql, "", "") + c.Assert(err, IsNil) + c.Assert(stmts, HasLen, 1) + + statements, err := SplitDDL(stmts[0], "test") + c.Assert(err, IsNil) + c.Assert(statements, DeepEquals, expectedSQLs[i]) + + tbs := expectedTableName[i] + for j, statement := range statements { + s, err := Parse(p, statement, "", "") + c.Assert(err, IsNil) + c.Assert(s, HasLen, 1) + + tableNames, err := FetchDDLTableNames("test", s[0]) + c.Assert(err, IsNil) + c.Assert(tableNames, DeepEquals, tbs[j]) + + targetSQL, err := RenameDDLTable(s[0], targetTableNames[i][j]) + c.Assert(err, IsNil) + c.Assert(targetSQL, Equals, targetSQLs[i][j]) + } + } + +} diff --git a/pkg/ddl/common.go b/pkg/parser/common.go similarity index 91% rename from pkg/ddl/common.go rename to pkg/parser/common.go index d1f19cc240..6cc25cf1cf 100644 --- a/pkg/ddl/common.go +++ b/pkg/parser/common.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ddl +package parser import ( "bytes" @@ -23,6 +23,7 @@ import ( "github.com/pingcap/parser/format" "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/filter" + _ "github.com/pingcap/tidb/types/parser_driver" ) // Parse wraps parser.Parse(), makes `parser` suitable for dm @@ -43,7 +44,7 @@ func Parse(p *parser.Parser, sql, charset, collation string) (stmt []ast.StmtNod // FetchDDLTableNames returns table names in ddl // the result contains [tableName] excepted create table like and rename table // for `create table like` DDL, result contains [sourceTableName, sourceRefTableName] -// for rename table ddl, result contains [targetOldTableName, sourceNewTableName] +// for rename table ddl, result contains [oldTableName, newTableName] func FetchDDLTableNames(schema string, stmt ast.StmtNode) ([]*filter.Table, error) { var res []*filter.Table switch v := stmt.(type) { @@ -107,7 +108,7 @@ func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string } case *ast.DropTableStmt: - if len(v.Tables) > 0 { + if len(v.Tables) > 1 { return "", errors.New("not allow operation: delete multiple tables in one statement") } @@ -125,7 +126,7 @@ func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema) v.Table.Name = model.NewCIStr(targetTableNames[0].Name) case *ast.RenameTableStmt: - if len(v.TableToTables) > 0 { + if len(v.TableToTables) > 1 { return "", errors.New("not allow operation: rename multiple tables in one statement") } @@ -135,13 +136,18 @@ func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string v.TableToTables[0].NewTable.Name = model.NewCIStr(targetTableNames[1].Name) case *ast.AlterTableStmt: - if len(v.Specs) > 0 { + if len(v.Specs) > 1 { return "", errors.New("not allow operation: rename multiple tables in one statement") } v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema) v.Table.Name = model.NewCIStr(targetTableNames[0].Name) + if v.Specs[0].Tp == ast.AlterTableRenameTable { + v.Specs[0].NewTable.Schema = model.NewCIStr(targetTableNames[1].Schema) + v.Specs[0].NewTable.Name = model.NewCIStr(targetTableNames[1].Name) + } + default: return "", errors.Errorf("unkown type ddl %+v", stmt) } @@ -178,11 +184,25 @@ func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { v.IfExists = true case *ast.DropTableStmt: v.IfExists = true - for _, t := range v.Tables { + + tables := v.Tables + for _, t := range tables { if t.Schema.O == "" { t.Schema = schemaName } + + v.Tables = []*ast.TableName{t} + bf.Reset() + err = stmt.Restore(ctx) + if err != nil { + return nil, errors.Annotate(err, "restore ast node") + } + + sqls = append(sqls, bf.String()) } + v.Tables = tables + + return sqls, nil case *ast.CreateTableStmt: v.IfNotExists = true if v.Table.Schema.O == "" { diff --git a/relay/util.go b/relay/util.go index 9330ca8a8a..b4fdf9038a 100644 --- a/relay/util.go +++ b/relay/util.go @@ -21,7 +21,7 @@ import ( "github.com/pingcap/parser/ast" "github.com/siddontang/go-mysql/replication" - "github.com/pingcap/dm/pkg/ddl" + parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/dm/pkg/utils" ) @@ -30,7 +30,7 @@ func checkIsDDL(sql string, p *parser.Parser) bool { sql = utils.TrimCtrlChars(sql) // if parse error, treat it as not a DDL - stmts, err := ddl.Parse(p, sql, "", "") + stmts, err := parserpkg.Parse(p, sql, "", "") if err != nil || len(stmts) == 0 { return false } diff --git a/syncer/ddl.go b/syncer/ddl.go index a7fc69f297..2e551ff529 100644 --- a/syncer/ddl.go +++ b/syncer/ddl.go @@ -14,8 +14,8 @@ package syncer import ( - ddlpkg "github.com/pingcap/dm/pkg/ddl" "github.com/pingcap/dm/pkg/log" + parsepkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -66,7 +66,7 @@ func (s *Syncer) parseDDLSQL(sql string, p *parser.Parser, schema string) (resul // We use Parse not ParseOneStmt here, because sometimes we got a commented out ddl which can't be parsed // by ParseOneStmt(it's a limitation of tidb parser.) - stmts, err := ddlpkg.Parse(p, sql, "", "") + stmts, err := parsepkg.Parse(p, sql, "", "") if err != nil { // log error rather than fatal, so other defer can be executed log.Errorf(IncompatibleDDLFormat, sql) @@ -131,7 +131,7 @@ func (s *Syncer) parseDDLSQL(sql string, p *parser.Parser, schema string) (resul // * try to apply online ddl by given online // return @spilted sqls, @online ddl table names, @error func (s *Syncer) resolveDDLSQL(p *parser.Parser, stmt ast.StmtNode, schema string) (sqls []string, tables map[string]*filter.Table, err error) { - sqls, err = ddlpkg.SplitDDL(stmt, schema) + sqls, err = parsepkg.SplitDDL(stmt, schema) if err != nil { return nil, nil, errors.Trace(err) } @@ -163,7 +163,7 @@ func (s *Syncer) handleDDL(p *parser.Parser, schema, sql string) (string, [][]*f return "", nil, nil, errors.Annotatef(err, "ddl %s", sql) } - tableNames, err := ddlpkg.FetchDDLTableNames(schema, stmt) + tableNames, err := parsepkg.FetchDDLTableNames(schema, stmt) if err != nil { return "", nil, nil, errors.Trace(err) } @@ -186,7 +186,7 @@ func (s *Syncer) handleDDL(p *parser.Parser, schema, sql string) (string, [][]*f targetTableNames = append(targetTableNames, tableName) } - ddl, err := ddlpkg.RenameDDLTable(stmt, targetTableNames) + ddl, err := parsepkg.RenameDDLTable(stmt, targetTableNames) return ddl, [][]*filter.Table{tableNames, targetTableNames}, stmt, errors.Trace(err) } @@ -202,7 +202,7 @@ func (s *Syncer) handleOnlineDDL(p *parser.Parser, schema, sql string) ([]string return nil, nil, errors.Annotatef(err, "ddl %s", sql) } - tableNames, err := ddlpkg.FetchDDLTableNames(schema, stmt) + tableNames, err := parsepkg.FetchDDLTableNames(schema, stmt) if err != nil { return nil, nil, errors.Trace(err) } @@ -227,7 +227,7 @@ func (s *Syncer) handleOnlineDDL(p *parser.Parser, schema, sql string) ([]string return nil, nil, errors.Trace(err) } - sqls[i], err = ddlpkg.RenameDDLTable(stmt, targetTables) + sqls[i], err = parsepkg.RenameDDLTable(stmt, targetTables) if err != nil { return nil, nil, errors.Trace(err) } diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index b158959fea..aa1f3063fa 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -20,151 +20,12 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/tidb-tools/pkg/filter" + parsepkg "github.com/pingcap/dm/pkg/parser" + "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/pkg/utils" ) -func (s *testSyncerSuite) TestFindTableDefineIndex(c *C) { - testCase := [][]string{ - {"create table t (id", "(id"}, - {"create table t(id", "(id"}, - {"create table t ( id", "( id"}, - {"create table t( id", "( id"}, - {"create table t", ""}, - } - - for _, t := range testCase { - c.Assert(findTableDefineIndex(t[0]), Equals, t[1]) - } -} - -func (s *testSyncerSuite) TestFindLastWord(c *C) { - testCase := [][]interface{}{ - {"create table t (id", 15}, - {"create table t(id", 13}, - {"create table t ( id", 17}, - {"create table t( id", 16}, - {"create table t", 13}, - } - - for _, t := range testCase { - c.Assert(findLastWord(t[0].(string)), Equals, t[1]) - } -} - -func (s *testSyncerSuite) TestGenDDLSQL(c *C) { - originTableNameSingle := []*filter.Table{ - {Schema: "test", Name: "test"}, - } - originTableNameDouble := []*filter.Table{ - {Schema: "test", Name: "test"}, - {Schema: "test1", Name: "test1"}, - } - targetTableNameSingle := []*filter.Table{{Schema: "titi", Name: "titi"}} - targetTableNameDouble := []*filter.Table{ - {Schema: "titi", Name: "titi"}, - {Schema: "titi1", Name: "titi1"}, - } - testCase := [][]string{ - {"CREATE DATABASE test", "CREATE DATABASE test", "CREATE DATABASE `titi`"}, - {"CREATE SCHEMA test", "CREATE SCHEMA test", "CREATE SCHEMA `titi`"}, - {"CREATE DATABASE IF NOT EXISTS test", "CREATE DATABASE IF NOT EXISTS test", "CREATE DATABASE IF NOT EXISTS `titi`"}, - {"DROP DATABASE test", "DROP DATABASE test", "DROP DATABASE `titi`"}, - {"DROP SCHEMA test", "DROP SCHEMA test", "DROP SCHEMA `titi`"}, - {"DROP DATABASE IF EXISTS test", "DROP DATABASE IF EXISTS test", "DROP DATABASE IF EXISTS `titi`"}, - {"CREATE TABLE test(id int)", "CREATE TABLE `test`.`test`(id int)", "USE `titi`; CREATE TABLE `titi`.`titi`(id int);"}, - {"CREATE TABLE test (id int)", "CREATE TABLE `test`.`test` (id int)", "USE `titi`; CREATE TABLE `titi`.`titi` (id int);"}, - {"DROP TABLE test", "DROP TABLE `test`.`test`", "USE `titi`; DROP TABLE `titi`.`titi`;"}, - {"TRUNCATE TABLE test", "TRUNCATE TABLE `test`.`test`", "USE `titi`; TRUNCATE TABLE `titi`.`titi`;"}, - {"alter table test add column abc int", "ALTER TABLE `test`.`test` add column abc int", "USE `titi`; ALTER TABLE `titi`.`titi` add column abc int;"}, - {"CREATE INDEX `idx1` on test(id)", "CREATE INDEX `idx1` ON `test`.`test` (id)", "USE `titi`; CREATE INDEX `idx1` ON `titi`.`titi` (id);"}, - {"CREATE INDEX `idx1` on test (id)", "CREATE INDEX `idx1` ON `test`.`test` (id)", "USE `titi`; CREATE INDEX `idx1` ON `titi`.`titi` (id);"}, - {"DROP INDEX `idx1` on test", "DROP INDEX `idx1` ON `test`.`test`", "USE `titi`; DROP INDEX `idx1` ON `titi`.`titi`;"}, - } - for _, t := range testCase { - p, err := utils.GetParser(s.db, false) - c.Assert(err, IsNil) - stmt, err := p.ParseOneStmt(t[0], "", "") - c.Assert(err, IsNil) - sql, err := genDDLSQL(t[0], stmt, originTableNameSingle, targetTableNameSingle, true) - c.Assert(err, IsNil) - c.Assert(sql, Equals, t[2]) - } - - testCase = [][]string{ - {"rename table test to test1", "RENAME TABLE `test`.`test` TO `test1`.`test1`", "RENAME TABLE `titi`.`titi` TO `titi1`.`titi1`"}, - {"alter table test rename as test1", "ALTER TABLE `test`.`test` rename as `test1`.`test1`", "USE `titi`; ALTER TABLE `titi`.`titi` rename as `titi1`.`titi1`;"}, - {"create table test like test1", "create table `test`.`test` like `test1`.`test1`", "USE `titi`; create table `titi`.`titi` like `titi1`.`titi1`;"}, - } - for _, t := range testCase { - p, err := utils.GetParser(s.db, false) - c.Assert(err, IsNil) - stmt, err := p.ParseOneStmt(t[0], "", "") - c.Assert(err, IsNil) - sql, err := genDDLSQL(t[0], stmt, originTableNameDouble, targetTableNameDouble, true) - c.Assert(err, IsNil) - c.Assert(sql, Equals, t[2]) - } - -} - -func (s *testSyncerSuite) TestComment(c *C) { - originTableNameSingle := []*filter.Table{ - {Schema: "test", Name: "test"}, - } - originTableNameDouble := []*filter.Table{ - {Schema: "test", Name: "test"}, - {Schema: "test1", Name: "test1"}, - } - targetTableNameSingle := []*filter.Table{ - {Schema: "titi", Name: "titi"}, - } - targetTableNameDouble := []*filter.Table{ - {Schema: "titi", Name: "titi"}, - {Schema: "titi1", Name: "titi1"}, - } - testCase := [][]string{ - {"CREATE /* gh-ost */ DATABASE test", "CREATE /* gh-ost */ DATABASE `titi`"}, - {"CREATE /* gh-ost */ SCHEMA test", "CREATE /* gh-ost */ SCHEMA `titi`"}, - {"CREATE /* gh-ost */ DATABASE IF NOT EXISTS test", "CREATE /* gh-ost */ DATABASE IF NOT EXISTS `titi`"}, - {"DROP /* gh-ost */ DATABASE test", "DROP /* gh-ost */ DATABASE `titi`"}, - {"DROP /* gh-ost */ SCHEMA test", "DROP /* gh-ost */ SCHEMA `titi`"}, - {"DROP /* gh-ost */ DATABASE IF EXISTS test", "DROP /* gh-ost */ DATABASE IF EXISTS `titi`"}, - {"CREATE /* gh-ost */ TABLE test(id int)", "USE `titi`; CREATE /* gh-ost */ TABLE `titi`.`titi`(id int);"}, - {"CREATE /* gh-ost */ TABLE test (id int)", "USE `titi`; CREATE /* gh-ost */ TABLE `titi`.`titi` (id int);"}, - {"DROP /* gh-ost */ TABLE test", "USE `titi`; DROP /* gh-ost */ TABLE `titi`.`titi`;"}, - {"TRUNCATE TABLE test", "USE `titi`; TRUNCATE TABLE `titi`.`titi`;"}, - {"alter /* gh-ost */ table test add column abc int", "USE `titi`; ALTER TABLE `titi`.`titi` add column abc int;"}, - {"CREATE /* gh-ost*/ INDEX `idx1` on test(id)", "USE `titi`; CREATE /* gh-ost*/ INDEX `idx1` ON `titi`.`titi` (id);"}, - {"CREATE /*gh-ost */ INDEX `idx1` on test (id)", "USE `titi`; CREATE /*gh-ost */ INDEX `idx1` ON `titi`.`titi` (id);"}, - {"DROP /*gh-ost*/ INDEX `idx1` on test", "USE `titi`; DROP /*gh-ost*/ INDEX `idx1` ON `titi`.`titi`;"}, - } - - parser, err := utils.GetParser(s.db, false) - c.Assert(err, IsNil) - - for _, t := range testCase { - stmt, err := parser.ParseOneStmt(t[0], "", "") - c.Assert(err, IsNil) - sql, err := genDDLSQL(t[0], stmt, originTableNameSingle, targetTableNameSingle, true) - c.Assert(err, IsNil) - c.Assert(sql, Equals, t[1]) - } - - testCase = [][]string{ - {"rename table test to test1", "RENAME TABLE `titi`.`titi` TO `titi1`.`titi1`"}, - {"alter /* gh-ost */ table test rename as test1", "USE `titi`; ALTER TABLE `titi`.`titi` rename as `titi1`.`titi1`;"}, - {"create /* gh-ost */ table test like test1", "USE `titi`; create /* gh-ost */ table `titi`.`titi` like `titi1`.`titi1`;"}, - } - for _, t := range testCase { - stmt, err := parser.ParseOneStmt(t[0], "", "") - c.Assert(err, IsNil) - sql, err := genDDLSQL(t[0], stmt, originTableNameDouble, targetTableNameDouble, true) - c.Assert(err, IsNil) - c.Assert(sql, Equals, t[1]) - } -} - func (s *testSyncerSuite) TestTrimCtrlChars(c *C) { ddl := "create table if not exists foo.bar(id int)" controlChars := make([]byte, 0, 33) @@ -194,6 +55,7 @@ func (s *testSyncerSuite) TestTrimCtrlChars(c *C) { buf.Reset() } } + func (s *testSyncerSuite) TestAnsiQuotes(c *C) { ansiQuotesCases := []string{ "create database `test`", @@ -240,7 +102,7 @@ CREATE TABLE test.test_table_with_c (id int); parser, err := utils.GetParser(s.db, false) c.Assert(err, IsNil) - _, err = parser.Parse(sql, "", "") + _, err = parsepkg.Parse(sql, "", "") c.Assert(err, IsNil) } @@ -251,18 +113,14 @@ func (s *testSyncerSuite) TestCommentQuote(c *C) { parser, err := utils.GetParser(s.db, false) c.Assert(err, IsNil) - _, err = parser.ParseOneStmt(sql, "", "") + stmt1, err = parser.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) syncer := &Syncer{} - sqls, _, _, err := syncer.resolveDDLSQL(sql, parser, "") + sqls, _, err := syncer.resolveDDLSQL(sql, parser, "") c.Assert(err, IsNil) c.Assert(len(sqls), Equals, 1) - - getSQL := sqls[0] - _, err = parser.ParseOneStmt(getSQL, "", "") - c.Assert(err, IsNil) - c.Assert(getSQL, Equals, expectedSQL) + c.Assert(sqls[0], Equals, expectedSQL) } func (s *testSyncerSuite) TestIgnoreDMLInQuery(c *C) { diff --git a/syncer/inject_sql.go b/syncer/inject_sql.go index 3268d6fb4b..48743ef526 100644 --- a/syncer/inject_sql.go +++ b/syncer/inject_sql.go @@ -16,8 +16,8 @@ package syncer import ( "time" - ddlpkg "github.com/pingcap/dm/pkg/ddl" "github.com/pingcap/dm/pkg/log" + parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -42,7 +42,7 @@ func (s *Syncer) InjectSQLs(ctx context.Context, sqls []string) error { if !ok { return errors.Errorf("only support inject DDL for sharding group to be synced currently, but got %s", sql) } - tableNames, err := ddlpkg.FetchDDLTableNames("", ddlNode) + tableNames, err := parserpkg.FetchDDLTableNames("", ddlNode) if err != nil { return errors.Trace(err) } diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index 047863fd64..cd1d86d10e 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -24,6 +24,7 @@ import ( _ "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" "github.com/pingcap/dm/pkg/log" + parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/parser/ast" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" cm "github.com/pingcap/tidb-tools/pkg/column-mapping" @@ -180,7 +181,7 @@ func (s *testSyncerSuite) TestSelectDB(c *C) { stmt, err := p.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) - tableNames, err := fetchDDLTableNames(string(ev.Schema), stmt) + tableNames, err := parserpkg.FetchDDLTableNames(string(ev.Schema), stmt) c.Assert(err, IsNil) r, err := syncer.skipQuery(tableNames, stmt, sql) @@ -272,7 +273,7 @@ func (s *testSyncerSuite) TestSelectTable(c *C) { switch ev := e.Event.(type) { case *replication.QueryEvent: query := string(ev.Query) - querys, _, _, err := syncer.resolveDDLSQL(query, p, string(ev.Schema)) + querys, _, err := syncer.resolveDDLSQL(query, p, string(ev.Schema)) c.Assert(err, IsNil) if len(querys) == 0 { continue @@ -282,7 +283,7 @@ func (s *testSyncerSuite) TestSelectTable(c *C) { stmt, err := p.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) - tableNames, err := fetchDDLTableNames(string(ev.Schema), stmt) + tableNames, err := parserpkg.FetchDDLTableNames(string(ev.Schema), stmt) c.Assert(err, IsNil) r, err := syncer.skipQuery(tableNames, stmt, sql) c.Assert(err, IsNil) @@ -346,7 +347,7 @@ func (s *testSyncerSuite) TestIgnoreDB(c *C) { stmt, err := p.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) - tableNames, err := fetchDDLTableNames(sql, stmt) + tableNames, err := parserpkg.FetchDDLTableNames(sql, stmt) c.Assert(err, IsNil) r, err := syncer.skipQuery(tableNames, stmt, sql) c.Assert(err, IsNil) @@ -430,7 +431,7 @@ func (s *testSyncerSuite) TestIgnoreTable(c *C) { switch ev := e.Event.(type) { case *replication.QueryEvent: query := string(ev.Query) - querys, _, _, err := syncer.resolveDDLSQL(query, p, string(ev.Schema)) + querys, _, err := syncer.resolveDDLSQL(query, p, string(ev.Schema)) c.Assert(err, IsNil) if len(querys) == 0 { continue @@ -440,7 +441,7 @@ func (s *testSyncerSuite) TestIgnoreTable(c *C) { stmt, err := p.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) - tableNames, err := fetchDDLTableNames(string(ev.Schema), stmt) + tableNames, err := parserpkg.FetchDDLTableNames(string(ev.Schema), stmt) c.Assert(err, IsNil) r, err := syncer.skipQuery(tableNames, stmt, sql) c.Assert(err, IsNil) From e5cd908a605b976b0326975ed7e09c922b82d766 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Wed, 20 Feb 2019 18:52:13 +0800 Subject: [PATCH 05/20] fix test --- syncer/ddl.go | 14 +++++++------- syncer/ddl_test.go | 12 +++++------- syncer/syncer_test.go | 10 ++++++++-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/syncer/ddl.go b/syncer/ddl.go index 2e551ff529..48e347e72c 100644 --- a/syncer/ddl.go +++ b/syncer/ddl.go @@ -15,7 +15,7 @@ package syncer import ( "github.com/pingcap/dm/pkg/log" - parsepkg "github.com/pingcap/dm/pkg/parser" + parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -66,7 +66,7 @@ func (s *Syncer) parseDDLSQL(sql string, p *parser.Parser, schema string) (resul // We use Parse not ParseOneStmt here, because sometimes we got a commented out ddl which can't be parsed // by ParseOneStmt(it's a limitation of tidb parser.) - stmts, err := parsepkg.Parse(p, sql, "", "") + stmts, err := parserpkg.Parse(p, sql, "", "") if err != nil { // log error rather than fatal, so other defer can be executed log.Errorf(IncompatibleDDLFormat, sql) @@ -131,7 +131,7 @@ func (s *Syncer) parseDDLSQL(sql string, p *parser.Parser, schema string) (resul // * try to apply online ddl by given online // return @spilted sqls, @online ddl table names, @error func (s *Syncer) resolveDDLSQL(p *parser.Parser, stmt ast.StmtNode, schema string) (sqls []string, tables map[string]*filter.Table, err error) { - sqls, err = parsepkg.SplitDDL(stmt, schema) + sqls, err = parserpkg.SplitDDL(stmt, schema) if err != nil { return nil, nil, errors.Trace(err) } @@ -163,7 +163,7 @@ func (s *Syncer) handleDDL(p *parser.Parser, schema, sql string) (string, [][]*f return "", nil, nil, errors.Annotatef(err, "ddl %s", sql) } - tableNames, err := parsepkg.FetchDDLTableNames(schema, stmt) + tableNames, err := parserpkg.FetchDDLTableNames(schema, stmt) if err != nil { return "", nil, nil, errors.Trace(err) } @@ -186,7 +186,7 @@ func (s *Syncer) handleDDL(p *parser.Parser, schema, sql string) (string, [][]*f targetTableNames = append(targetTableNames, tableName) } - ddl, err := parsepkg.RenameDDLTable(stmt, targetTableNames) + ddl, err := parserpkg.RenameDDLTable(stmt, targetTableNames) return ddl, [][]*filter.Table{tableNames, targetTableNames}, stmt, errors.Trace(err) } @@ -202,7 +202,7 @@ func (s *Syncer) handleOnlineDDL(p *parser.Parser, schema, sql string) ([]string return nil, nil, errors.Annotatef(err, "ddl %s", sql) } - tableNames, err := parsepkg.FetchDDLTableNames(schema, stmt) + tableNames, err := parserpkg.FetchDDLTableNames(schema, stmt) if err != nil { return nil, nil, errors.Trace(err) } @@ -227,7 +227,7 @@ func (s *Syncer) handleOnlineDDL(p *parser.Parser, schema, sql string) ([]string return nil, nil, errors.Trace(err) } - sqls[i], err = parsepkg.RenameDDLTable(stmt, targetTables) + sqls[i], err = parserpkg.RenameDDLTable(stmt, targetTables) if err != nil { return nil, nil, errors.Trace(err) } diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index aa1f3063fa..12c450af88 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -18,12 +18,10 @@ import ( "database/sql" . "github.com/pingcap/check" - "github.com/pingcap/tidb-tools/pkg/filter" - - parsepkg "github.com/pingcap/dm/pkg/parser" - "github.com/pingcap/dm/dm/config" + parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/dm/pkg/utils" + "github.com/pingcap/tidb-tools/pkg/filter" ) func (s *testSyncerSuite) TestTrimCtrlChars(c *C) { @@ -102,7 +100,7 @@ CREATE TABLE test.test_table_with_c (id int); parser, err := utils.GetParser(s.db, false) c.Assert(err, IsNil) - _, err = parsepkg.Parse(sql, "", "") + _, err = parserpkg.Parse(parser, sql, "", "") c.Assert(err, IsNil) } @@ -113,11 +111,11 @@ func (s *testSyncerSuite) TestCommentQuote(c *C) { parser, err := utils.GetParser(s.db, false) c.Assert(err, IsNil) - stmt1, err = parser.ParseOneStmt(sql, "", "") + stmt, err = parser.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) syncer := &Syncer{} - sqls, _, err := syncer.resolveDDLSQL(sql, parser, "") + sqls, _, err := syncer.resolveDDLSQL(parser, stmt, "schemadb") c.Assert(err, IsNil) c.Assert(len(sqls), Equals, 1) c.Assert(sqls[0], Equals, expectedSQL) diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index cd1d86d10e..7d04d07c57 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -273,7 +273,10 @@ func (s *testSyncerSuite) TestSelectTable(c *C) { switch ev := e.Event.(type) { case *replication.QueryEvent: query := string(ev.Query) - querys, _, err := syncer.resolveDDLSQL(query, p, string(ev.Schema)) + stmt, err = parser.ParseOneStmt(query, "", "") + c.Assert(err, IsNil) + + querys, _, err := syncer.resolveDDLSQL(p, stmt, string(ev.Schema)) c.Assert(err, IsNil) if len(querys) == 0 { continue @@ -431,7 +434,10 @@ func (s *testSyncerSuite) TestIgnoreTable(c *C) { switch ev := e.Event.(type) { case *replication.QueryEvent: query := string(ev.Query) - querys, _, err := syncer.resolveDDLSQL(query, p, string(ev.Schema)) + stmt, err = parser.ParseOneStmt(query, "", "") + c.Assert(err, IsNil) + + querys, _, err := syncer.resolveDDLSQL(p, stmt, string(ev.Schema)) c.Assert(err, IsNil) if len(querys) == 0 { continue From 67f23fef72b683a06e8c730aed29b629ebd79713 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Wed, 20 Feb 2019 18:57:30 +0800 Subject: [PATCH 06/20] fix test --- syncer/ddl_test.go | 2 +- syncer/syncer_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index 12c450af88..1ab3fca942 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -111,7 +111,7 @@ func (s *testSyncerSuite) TestCommentQuote(c *C) { parser, err := utils.GetParser(s.db, false) c.Assert(err, IsNil) - stmt, err = parser.ParseOneStmt(sql, "", "") + stmt, err := parser.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) syncer := &Syncer{} diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index 7d04d07c57..5733d4ebcf 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -273,7 +273,7 @@ func (s *testSyncerSuite) TestSelectTable(c *C) { switch ev := e.Event.(type) { case *replication.QueryEvent: query := string(ev.Query) - stmt, err = parser.ParseOneStmt(query, "", "") + stmt, err := p.ParseOneStmt(query, "", "") c.Assert(err, IsNil) querys, _, err := syncer.resolveDDLSQL(p, stmt, string(ev.Schema)) @@ -434,7 +434,7 @@ func (s *testSyncerSuite) TestIgnoreTable(c *C) { switch ev := e.Event.(type) { case *replication.QueryEvent: query := string(ev.Query) - stmt, err = parser.ParseOneStmt(query, "", "") + stmt, err := p.ParseOneStmt(query, "", "") c.Assert(err, IsNil) querys, _, err := syncer.resolveDDLSQL(p, stmt, string(ev.Schema)) From d51e638296770f3ece4990f69235449e5e0ca908 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Wed, 20 Feb 2019 19:14:29 +0800 Subject: [PATCH 07/20] fix test --- pkg/parser/common.go | 2 +- syncer/ddl_test.go | 2 +- syncer/syncer_test.go | 14 ++++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/parser/common.go b/pkg/parser/common.go index 6cc25cf1cf..f7e4296bca 100644 --- a/pkg/parser/common.go +++ b/pkg/parser/common.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/parser/format" "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/filter" - _ "github.com/pingcap/tidb/types/parser_driver" + _ "github.com/pingcap/tidb/types/parser_driver" // for import parser driver ) // Parse wraps parser.Parse(), makes `parser` suitable for dm diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index 1ab3fca942..ee8c0a77ca 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -106,7 +106,7 @@ CREATE TABLE test.test_table_with_c (id int); func (s *testSyncerSuite) TestCommentQuote(c *C) { sql := "ALTER TABLE schemadb.ep_edu_course_message_auto_reply MODIFY answer JSON COMMENT '回复的内容-格式为list，有两个字段：\"answerType\"：//''发送客服消息类型：1-文本消息，2-图片，3-图文链接''； answer：回复内容';" - expectedSQL := "ALTER TABLE `schemadb`.`ep_edu_course_message_auto_reply` MODIFY COLUMN `answer` json COMMENT '回复的内容-格式为list，有两个字段：\"answerType\"：//''发送客服消息类型：1-文本消息，2-图片，3-图文链接''； answer：回复内容'" + expectedSQL := "ALTER TABLE `schemadb`.`ep_edu_course_message_auto_reply` MODIFY COLUMN `answer` JSON COMMENT '回复的内容-格式为list，有两个字段：\"answerType\"：//''发送客服消息类型：1-文本消息，2-图片，3-图文链接''； answer：回复内容'" parser, err := utils.GetParser(s.db, false) c.Assert(err, IsNil) diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index 5733d4ebcf..15489b8dd8 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -273,10 +273,13 @@ func (s *testSyncerSuite) TestSelectTable(c *C) { switch ev := e.Event.(type) { case *replication.QueryEvent: query := string(ev.Query) - stmt, err := p.ParseOneStmt(query, "", "") + result, err := syncer.parseDDLSQL(query, p, string(ev.Schema)) c.Assert(err, IsNil) + if !result.isDDL { + continue // BEGIN event + } - querys, _, err := syncer.resolveDDLSQL(p, stmt, string(ev.Schema)) + querys, _, err := syncer.resolveDDLSQL(p, result.stmt, string(ev.Schema)) c.Assert(err, IsNil) if len(querys) == 0 { continue @@ -434,10 +437,13 @@ func (s *testSyncerSuite) TestIgnoreTable(c *C) { switch ev := e.Event.(type) { case *replication.QueryEvent: query := string(ev.Query) - stmt, err := p.ParseOneStmt(query, "", "") + result, err := syncer.parseDDLSQL(query, p, string(ev.Schema)) c.Assert(err, IsNil) + if !result.isDDL { + continue // BEGIN event + } - querys, _, err := syncer.resolveDDLSQL(p, stmt, string(ev.Schema)) + querys, _, err := syncer.resolveDDLSQL(p, result.stmt, string(ev.Schema)) c.Assert(err, IsNil) if len(querys) == 0 { continue From 4e8f000020c1cb5f2a27010939d622fd568c5154 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Wed, 20 Feb 2019 20:47:00 +0800 Subject: [PATCH 08/20] add test cases --- syncer/ddl_test.go | 131 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 130 insertions(+), 1 deletion(-) diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index ee8c0a77ca..2c4929f3ac 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -21,7 +21,9 @@ import ( "github.com/pingcap/dm/dm/config" parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/dm/pkg/utils" + "github.com/pingcap/parser" "github.com/pingcap/tidb-tools/pkg/filter" + router "github.com/pingcap/tidb-tools/pkg/table-router" ) func (s *testSyncerSuite) TestTrimCtrlChars(c *C) { @@ -121,7 +123,120 @@ func (s *testSyncerSuite) TestCommentQuote(c *C) { c.Assert(sqls[0], Equals, expectedSQL) } -func (s *testSyncerSuite) TestIgnoreDMLInQuery(c *C) { +func (s *testSyncerSuite) TestresolveDDLSQL(c *C) { + // duplicate with pkg/parser + sqls := []string{ + "create schema `s1`", + "create schema if not exists `s1`", + "drop schema `s1`", + "drop schema if exists `s1`", + "drop table `s1`.`t1`", + "drop table `s1`.`t1`, `s2`.`t2`", + "drop table `s1`.`t1`, `s2`.`t2`, `xx`", + "create table `s1`.`t1` (id int)", + "create table `t1` (id int)", + "create table `t1` like `t2`", + "create table `s1`.`t1` like `t2`", + "create table `t1` like `xx`.`t2`", + "truncate table `t1`", + "truncate table `s1`.`t1`", + "rename table `s1`.`t1` to `s2`.`t2`", + "rename table `t1` to `t2`, `s1`.`t1` to `t2`", + "drop index i1 on `s1`.`t1`", + "drop index i1 on `t1`", + "create index i1 on `t1`(`c1`)", + "create index i1 on `s1`.`t1`(`c1`)", + "alter table `t1` add column c1 int, drop column c2", + "alter table `s1`.`t1` add column c1 int, rename to `t2`, drop column c2", + "alter table `s1`.`t1` add column c1 int, rename to `xx`.`t2`, drop column c2", + } + + expectedSQLs := [][]string{ + {"CREATE DATABASE IF NOT EXISTS `s1`"}, + {"CREATE DATABASE IF NOT EXISTS `s1`"}, + {"DROP DATABASE IF EXISTS `s1`"}, + {"DROP DATABASE IF EXISTS `s1`"}, + {"DROP TABLE IF EXISTS `s1`.`t1`"}, + {"DROP TABLE IF EXISTS `s1`.`t1`", "DROP TABLE IF EXISTS `s2`.`t2`"}, + {"DROP TABLE IF EXISTS `s1`.`t1`", "DROP TABLE IF EXISTS `s2`.`t2`", "DROP TABLE IF EXISTS `test`.`xx`"}, + {"CREATE TABLE IF NOT EXISTS `s1`.`t1` (`id` INT)"}, + {"CREATE TABLE IF NOT EXISTS `test`.`t1` (`id` INT)"}, + {"CREATE TABLE IF NOT EXISTS `test`.`t1` LIKE `test`.`t2`"}, + {"CREATE TABLE IF NOT EXISTS `s1`.`t1` LIKE `test`.`t2`"}, + {"CREATE TABLE IF NOT EXISTS `test`.`t1` LIKE `xx`.`t2`"}, + {"TRUNCATE TABLE `test`.`t1`"}, + {"TRUNCATE TABLE `s1`.`t1`"}, + {"RENAME TABLE `s1`.`t1` TO `s2`.`t2`"}, + {"RENAME TABLE `test`.`t1` TO `test`.`t2`", "RENAME TABLE `s1`.`t1` TO `test`.`t2`"}, + {"DROP INDEX IF EXISTS `i1` ON `s1`.`t1`"}, + {"DROP INDEX IF EXISTS `i1` ON `test`.`t1`"}, + {"CREATE INDEX `i1` ON `test`.`t1` (`c1`)"}, + {"CREATE INDEX `i1` ON `s1`.`t1` (`c1`)"}, + {"ALTER TABLE `test`.`t1` ADD COLUMN `c1` INT", "ALTER TABLE `test`.`t1` DROP COLUMN `c2`"}, + {"ALTER TABLE `s1`.`t1` ADD COLUMN `c1` INT", "ALTER TABLE `s1`.`t1` RENAME AS `test`.`t2`", "ALTER TABLE `test`.`t2` DROP COLUMN `c2`"}, + {"ALTER TABLE `s1`.`t1` ADD COLUMN `c1` INT", "ALTER TABLE `s1`.`t1` RENAME AS `xx`.`t2`", "ALTER TABLE `xx`.`t2` DROP COLUMN `c2`"}, + } + + targetSQLs := [][]string{ + {"CREATE DATABASE IF NOT EXISTS `xs1`"}, + {"CREATE DATABASE IF NOT EXISTS `xs1`"}, + {"DROP DATABASE IF EXISTS `xs1`"}, + {"DROP DATABASE IF EXISTS `xs1`"}, + {"DROP TABLE IF EXISTS `xs1`.`t1`"}, + {"DROP TABLE IF EXISTS `xs1`.`t1`", ""}, + {"DROP TABLE IF EXISTS `xs1`.`t1`", "", ""}, + {"CREATE TABLE IF NOT EXISTS `xs1`.`t1` (`id` INT)"}, + {""}, + {""}, + {""}, + {""}, + {""}, + {"TRUNCATE TABLE `xs1`.`t1`"}, + {""}, + {"", ""}, + {"DROP INDEX IF EXISTS `i1` ON `xs1`.`t1`"}, + {""}, + {""}, + {"CREATE INDEX `i1` ON `xs1`.`t1` (`c1`)"}, + {"", ""}, + {"ALTER TABLE `xs1`.`t1` ADD COLUMN `c1` INT", "", ""}, + {"ALTER TABLE `xs1`.`t1` ADD COLUMN `c1` INT", "", ""}, + } + + p := parser.New() + cfg := &config.SubTaskConfig{ + BWList: &filter.Rules{ + DoDBs: []string{"s1"}, + }, + } + syncer := NewSyncer(cfg) + + syncer.tableRouter = router.NewTableRouter(false, []*router.TableRule{ + { + SchemaPattern: "s1", + TargetPattern: "xs1", + }, + }) + + for i, sql := range sqls { + result, err := syncer.parseDDLSQL(sql, p, "test") + c.Assert(err, IsNil) + c.Assert(result.ignore, IsFalse) + c.Assert(result.isDDL, IsTrue) + + statements, _, err := syncer.resolveDDLSQL(p, result.stmt, "test") + c.Assert(err, IsNil) + c.Assert(statements, DeepEquals, expectedSQLs[i]) + + for j, statement := range statements { + s, _, _, err := syncer.handleDDL(p, "test", statement) + c.Assert(err, IsNil) + c.Assert(s, Equals, targetSQLs[i][j]) + } + } +} + +func (s *testSyncerSuite) TestParseDDLSQL(c *C) { cases := []struct { sql string schema string @@ -129,6 +244,20 @@ func (s *testSyncerSuite) TestIgnoreDMLInQuery(c *C) { isDDL bool hasError bool }{ + { + sql: "FLUSH", + schema: "", + ignore: true, + isDDL: false, + hasError: false, + }, + { + sql: "BEGIN", + schema: "", + ignore: false, + isDDL: false, + hasError: false, + }, { sql: "CREATE TABLE do_db.do_table (c1 INT)", schema: "", From 2f21d546bc5cef7565663cb86bad4c82dfa24ab0 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Wed, 20 Feb 2019 20:57:29 +0800 Subject: [PATCH 09/20] fix test --- syncer/ddl_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index 2c4929f3ac..1bcd17a2b2 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -211,12 +211,14 @@ func (s *testSyncerSuite) TestresolveDDLSQL(c *C) { } syncer := NewSyncer(cfg) - syncer.tableRouter = router.NewTableRouter(false, []*router.TableRule{ + var err error + syncer.tableRouter, err = router.NewTableRouter(false, []*router.TableRule{ { SchemaPattern: "s1", - TargetPattern: "xs1", + TargetSchema: "xs1", }, }) + c.Assert(err, IsNil) for i, sql := range sqls { result, err := syncer.parseDDLSQL(sql, p, "test") From e860425d99b89e93d9d57634ef4d5dd261c91ff1 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Wed, 20 Feb 2019 21:23:06 +0800 Subject: [PATCH 10/20] *: fix test case --- syncer/filter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncer/filter.go b/syncer/filter.go index 36c6bf95b5..1304bfe6e6 100644 --- a/syncer/filter.go +++ b/syncer/filter.go @@ -110,7 +110,7 @@ func (s *Syncer) skipQuery(tables []*filter.Table, stmt ast.StmtNode, sql string if len(tables) > 0 { tbs := s.bwList.ApplyOn(tables) - if len(tbs) == 0 { + if len(tbs) != len(tables) { return true, nil } } From ad0674f7079539f9464acbe46302436e91f4c384 Mon Sep 17 00:00:00 2001 From: gregoryIan Date: Thu, 21 Feb 2019 10:09:22 +0800 Subject: [PATCH 11/20] refine parser pkg --- pkg/parser/common.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/pkg/parser/common.go b/pkg/parser/common.go index f7e4296bca..5ac550c399 100644 --- a/pkg/parser/common.go +++ b/pkg/parser/common.go @@ -26,6 +26,15 @@ import ( _ "github.com/pingcap/tidb/types/parser_driver" // for import parser driver ) +var ( + // ErrDropMultipleTables is error that don't allow to drop multiple tables in one statement + ErrDropMultipleTables = errors.New("not allow operation: drop multiple tables in one statement") + // ErrRenameMultipleTables is error that don't allow to rename multiple tables in one statement + ErrRenameMultipleTables = errors.New("not allow operation: rename multiple tables in one statement") + // ErrAlterMultipleTables is error that don't allow to alter multiple tables in one statement + ErrAlterMultipleTables = errors.New("not allow operation: rename multiple tables in one statement") +) + // Parse wraps parser.Parse(), makes `parser` suitable for dm func Parse(p *parser.Parser, sql, charset, collation string) (stmt []ast.StmtNode, err error) { stmts, warnings, err := p.Parse(sql, charset, collation) @@ -59,7 +68,7 @@ func FetchDDLTableNames(schema string, stmt ast.StmtNode) ([]*filter.Table, erro } case *ast.DropTableStmt: if len(v.Tables) != 1 { - return res, errors.Errorf("drop table with multiple tables, may resovle ddl sql failed") + return res, ErrDropMultipleTables } res = append(res, genTableName(v.Tables[0].Schema.O, v.Tables[0].Name.O)) case *ast.TruncateTableStmt: @@ -89,7 +98,8 @@ func FetchDDLTableNames(schema string, stmt ast.StmtNode) ([]*filter.Table, erro return res, nil } -// RenameDDLTable renames table names in ddl by given `targetNames` +// RenameDDLTable renames table names in ddl by given `targetTableNames` +// argument `targetTableNames` is same with return value of FetchDDLTableNames func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string, error) { switch v := stmt.(type) { case *ast.CreateDatabaseStmt: @@ -109,7 +119,7 @@ func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string case *ast.DropTableStmt: if len(v.Tables) > 1 { - return "", errors.New("not allow operation: delete multiple tables in one statement") + return "", ErrDropMultipleTables } v.Tables[0].Schema = model.NewCIStr(targetTableNames[0].Schema) @@ -127,7 +137,7 @@ func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string v.Table.Name = model.NewCIStr(targetTableNames[0].Name) case *ast.RenameTableStmt: if len(v.TableToTables) > 1 { - return "", errors.New("not allow operation: rename multiple tables in one statement") + return "", ErrRenameMultipleTables } v.TableToTables[0].OldTable.Schema = model.NewCIStr(targetTableNames[0].Schema) @@ -137,7 +147,7 @@ func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string case *ast.AlterTableStmt: if len(v.Specs) > 1 { - return "", errors.New("not allow operation: rename multiple tables in one statement") + return "", ErrAlterMultipleTables } v.Table.Schema = model.NewCIStr(targetTableNames[0].Schema) @@ -165,7 +175,8 @@ func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string return bf.String(), nil } -// SplitDDL split multiple operations in one DDL statement into multiple DDL statements +// SplitDDL splits multiple operations in one DDL statement into multiple DDL statements +// if fail to restore, it would not restore the value of `stmt` (it changes it's values if `stmt` is one of DropTableStmt, RenameTableStmt, AlterTableStmt) func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { var ( b []byte From bfa59cb9bd759cad25b05c1380aa99926a351ae1 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Mon, 25 Feb 2019 15:25:09 +0800 Subject: [PATCH 12/20] *: fix test case --- syncer/ddl_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index ec20becc30..a537b31769 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -350,7 +350,7 @@ func (s *testSyncerSuite) TestResolveGeneratedColumnSQL(c *C) { ast1, err := parser.ParseOneStmt(tc.sql, "", "") c.Assert(err, IsNil) - sqls, _, _, err := syncer.resolveDDLSQL(tc.sql, parser, "") + sqls, _, err := syncer.resolveDDLSQL(tc.sql, parser, "") c.Assert(err, IsNil) c.Assert(len(sqls), Equals, 1) From 7365af387ae96a3eeb019c33e1b8d5d4e65ce69d Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Mon, 25 Feb 2019 15:35:42 +0800 Subject: [PATCH 13/20] *: fix test case --- syncer/ddl_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index a537b31769..fc9697fb80 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -350,7 +350,7 @@ func (s *testSyncerSuite) TestResolveGeneratedColumnSQL(c *C) { ast1, err := parser.ParseOneStmt(tc.sql, "", "") c.Assert(err, IsNil) - sqls, _, err := syncer.resolveDDLSQL(tc.sql, parser, "") + sqls, _, err := syncer.resolveDDLSQL(parser, ast1, "test") c.Assert(err, IsNil) c.Assert(len(sqls), Equals, 1) From 35ede2e7ab76cd1e04c00d8339d39ca61ad042d9 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Mon, 25 Feb 2019 15:52:38 +0800 Subject: [PATCH 14/20] *: fix test case --- syncer/ddl_test.go | 4 +- syncer/filter.go | 4 ++ test.go | 119 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 test.go diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index fc9697fb80..fc7d2ea801 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -334,11 +334,11 @@ func (s *testSyncerSuite) TestResolveGeneratedColumnSQL(c *C) { }{ { "ALTER TABLE `test`.`test` ADD COLUMN d int(11) GENERATED ALWAYS AS (c + 1) VIRTUAL", - "ALTER TABLE `test`.`test` ADD COLUMN `d` int(11) GENERATED ALWAYS AS (c + 1) VIRTUAL", + "ALTER TABLE `test`.`test` ADD COLUMN `d` INT(11) GENERATED ALWAYS AS(`c`+1)", }, { "ALTER TABLE `test`.`test` ADD COLUMN d int(11) AS (1 + 1) STORED", - "ALTER TABLE `test`.`test` ADD COLUMN `d` int(11) GENERATED ALWAYS AS (1 + 1) STORED", + "ALTER TABLE `test`.`test` ADD COLUMN `d` INT(11) GENERATED ALWAYS AS(1+1)", }, } diff --git a/syncer/filter.go b/syncer/filter.go index 6d9dedf0b7..1304bfe6e6 100644 --- a/syncer/filter.go +++ b/syncer/filter.go @@ -30,6 +30,10 @@ CREATE [TEMPORARY] TABLE [IF NOT EXISTS] tbl_name */ var ( builtInSkipDDLs = []string{ + // For mariadb, for query event, like `# Dumm` + // But i don't know what is the meaning of this event. + "^#", + // transaction "^SAVEPOINT", diff --git a/test.go b/test.go new file mode 100644 index 0000000000..27fa23d313 --- /dev/null +++ b/test.go @@ -0,0 +1,119 @@ +package main + +import ( + "bytes" + "fmt" + + "github.com/pingcap/parser" + "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/format" + "github.com/pingcap/parser/model" + _ "github.com/pingcap/tidb/types/parser_driver" +) + +func main() { + sqls := []string{ + "ALTER TABLE `test`.`test` ADD COLUMN d int(11) GENERATED ALWAYS AS (c + 1) VIRTUAL", + "ALTER TABLE `test`.`test` ADD COLUMN d int(11) AS (1 + 1) STORED", + } + + var b []byte + bf := bytes.NewBuffer(b) + schemaName := model.NewCIStr("test") + ctx := &format.RestoreCtx{ + Flags: format.DefaultRestoreFlags, + In: bf, + } + + for _, sql := range sqls { + fmt.Println("\n###################################################") + fmt.Println("handle", sql) + stmts, warnings, err := parser.New().Parse(sql, "", "") + fmt.Println("result:", warnings, err) + + for _, stmt := range stmts { + switch v := stmt.(type) { + case *ast.CreateDatabaseStmt: + v.IfNotExists = true + case *ast.DropDatabaseStmt: + v.IfExists = true + case *ast.DropTableStmt: + v.IfExists = true + for _, t := range v.Tables { + if t.Schema.O == "" { + t.Schema = schemaName + } + } + case *ast.CreateTableStmt: + v.IfNotExists = true + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + + if v.ReferTable != nil && v.ReferTable.Schema.O == "" { + v.ReferTable.Schema = schemaName + } + case *ast.TruncateTableStmt: + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + case *ast.DropIndexStmt: + v.IfExists = true + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + case *ast.CreateIndexStmt: + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + case *ast.RenameTableStmt: + t2ts := v.TableToTables + for _, t2t := range t2ts { + if t2t.OldTable.Schema.O == "" { + t2t.OldTable.Schema = schemaName + } + if t2t.NewTable.Schema.O == "" { + t2t.NewTable.Schema = schemaName + } + + v.TableToTables = []*ast.TableToTable{t2t} + + bf.Reset() + stmt.Restore(ctx) + fmt.Println(bf.String()) + } + continue + case *ast.AlterTableStmt: + specs := v.Specs + + if v.Table.Schema.O == "" { + v.Table.Schema = schemaName + } + + for _, spec := range specs { + if spec.Tp == ast.AlterTableRenameTable { + if spec.NewTable.Schema.O == "" { + spec.NewTable.Schema = schemaName + } + } + + v.Specs = []*ast.AlterTableSpec{spec} + + bf.Reset() + stmt.Restore(ctx) + fmt.Println(bf.String()) + + if spec.Tp == ast.AlterTableRenameTable { + v.Table = spec.NewTable + } + } + + continue + } + + bf.Reset() + stmt.Restore(ctx) + fmt.Println(bf.String()) + } + } +} From 4401acf280a57262934fd2b8d78b37393a52ae82 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Mon, 25 Feb 2019 16:03:21 +0800 Subject: [PATCH 15/20] *: fix test --- syncer/ddl_test.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index fc7d2ea801..3a9a031310 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -22,7 +22,6 @@ import ( parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/dm/pkg/utils" "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" ) @@ -357,15 +356,18 @@ func (s *testSyncerSuite) TestResolveGeneratedColumnSQL(c *C) { getSQL := sqls[0] c.Assert(getSQL, Equals, tc.expected) - ast2, err := parser.ParseOneStmt(getSQL, "", "") - c.Assert(err, IsNil) + // FIXME: remove me after implement generated function in restore + /* + ast2, err := parser.ParseOneStmt(getSQL, "", "") + c.Assert(err, IsNil) - // compare parsed ast of the resoved SQL with parsed ast of the origin SQL. - // because text fields are not always same, and the difference of text - // makes no sense to the semantics, we just ignore checking it. - atStmt1 := ast1.(*ast.AlterTableStmt) - atStmt2 := ast2.(*ast.AlterTableStmt) - c.Assert(atStmt1.Table, DeepEquals, atStmt2.Table) - c.Assert(atStmt1.Specs, DeepEquals, atStmt2.Specs) + // compare parsed ast of the resoved SQL with parsed ast of the origin SQL. + // because text fields are not always same, and the difference of text + // makes no sense to the semantics, we just ignore checking it. + atStmt1 := ast1.(*ast.AlterTableStmt) + atStmt2 := ast2.(*ast.AlterTableStmt) + c.Assert(atStmt1.Table, DeepEquals, atStmt2.Table) + c.Assert(atStmt1.Specs, DeepEquals, atStmt2.Specs) + */ } } From cc2583f5db9225cca1af2fd4cb7ea04b37b6ee50 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Tue, 26 Feb 2019 17:58:23 +0800 Subject: [PATCH 16/20] *:remove fix me --- go.mod | 2 +- go.sum | 4 ++-- syncer/ddl_test.go | 26 ++++++++++++-------------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index f733f3f3c9..341f20265a 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 github.com/pingcap/errors v0.11.0 - github.com/pingcap/parser v0.0.0-20190219084020-362712b5ab93 + github.com/pingcap/parser v0.0.0-20190225125525-01444ee2a9c2 github.com/pingcap/tidb v0.0.0-20190219092241-45ee2058969c github.com/pingcap/tidb-tools v2.1.3-0.20190218062145-515a74c8fb26+incompatible github.com/prometheus/client_golang v0.9.1 diff --git a/go.sum b/go.sum index a32d28efdc..e96d71d4b0 100644 --- a/go.sum +++ b/go.sum @@ -128,8 +128,8 @@ github.com/pingcap/kvproto v0.0.0-20190131052532-7e329e0c9e32/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7 h1:kOHAMalwF69bJrtWrOdVaCSvZjLucrJhP4NQKIu6uM4= github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/parser v0.0.0-20190218033509-9545f168ae97/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/parser v0.0.0-20190219084020-362712b5ab93 h1:jRnleVTpcisLEHNRYHBu4ilwEPyUkslJJfhYNEQlz2g= -github.com/pingcap/parser v0.0.0-20190219084020-362712b5ab93/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190225125525-01444ee2a9c2 h1:6M8bHGK2DLJBq4LfN7NNVOBhW2CvXHquyUVOuwjlZrY= +github.com/pingcap/parser v0.0.0-20190225125525-01444ee2a9c2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE= github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/tidb v0.0.0-20190219092241-45ee2058969c h1:1BlUrIxyx/nJ2C3Cr5AbFHjeGloqZgfp7dYeojuP2FI= diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index 3a9a031310..a805a1f075 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -22,6 +22,7 @@ import ( parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/dm/pkg/utils" "github.com/pingcap/parser" + "github.com/pingcap/parser/ast" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" ) @@ -333,11 +334,11 @@ func (s *testSyncerSuite) TestResolveGeneratedColumnSQL(c *C) { }{ { "ALTER TABLE `test`.`test` ADD COLUMN d int(11) GENERATED ALWAYS AS (c + 1) VIRTUAL", - "ALTER TABLE `test`.`test` ADD COLUMN `d` INT(11) GENERATED ALWAYS AS(`c`+1)", + "ALTER TABLE `test`.`test` ADD COLUMN `d` INT(11) GENERATED ALWAYS AS(`c`+1) VIRTUAL", }, { "ALTER TABLE `test`.`test` ADD COLUMN d int(11) AS (1 + 1) STORED", - "ALTER TABLE `test`.`test` ADD COLUMN `d` INT(11) GENERATED ALWAYS AS(1+1)", + "ALTER TABLE `test`.`test` ADD COLUMN `d` INT(11) GENERATED ALWAYS AS(1+1) STORED", }, } @@ -356,18 +357,15 @@ func (s *testSyncerSuite) TestResolveGeneratedColumnSQL(c *C) { getSQL := sqls[0] c.Assert(getSQL, Equals, tc.expected) - // FIXME: remove me after implement generated function in restore - /* - ast2, err := parser.ParseOneStmt(getSQL, "", "") - c.Assert(err, IsNil) + ast2, err := parser.ParseOneStmt(getSQL, "", "") + c.Assert(err, IsNil) - // compare parsed ast of the resoved SQL with parsed ast of the origin SQL. - // because text fields are not always same, and the difference of text - // makes no sense to the semantics, we just ignore checking it. - atStmt1 := ast1.(*ast.AlterTableStmt) - atStmt2 := ast2.(*ast.AlterTableStmt) - c.Assert(atStmt1.Table, DeepEquals, atStmt2.Table) - c.Assert(atStmt1.Specs, DeepEquals, atStmt2.Specs) - */ + // compare parsed ast of the resoved SQL with parsed ast of the origin SQL. + // because text fields are not always same, and the difference of text + // makes no sense to the semantics, we just ignore checking it. + atStmt1 := ast1.(*ast.AlterTableStmt) + atStmt2 := ast2.(*ast.AlterTableStmt) + c.Assert(atStmt1.Table, DeepEquals, atStmt2.Table) + c.Assert(atStmt1.Specs, DeepEquals, atStmt2.Specs) } } From 0f526f7c4be1266cab6ba7521049122b813b5915 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Tue, 26 Feb 2019 17:59:05 +0800 Subject: [PATCH 17/20] remove useless file --- test.go | 119 -------------------------------------------------------- 1 file changed, 119 deletions(-) delete mode 100644 test.go diff --git a/test.go b/test.go deleted file mode 100644 index 27fa23d313..0000000000 --- a/test.go +++ /dev/null @@ -1,119 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - - "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/format" - "github.com/pingcap/parser/model" - _ "github.com/pingcap/tidb/types/parser_driver" -) - -func main() { - sqls := []string{ - "ALTER TABLE `test`.`test` ADD COLUMN d int(11) GENERATED ALWAYS AS (c + 1) VIRTUAL", - "ALTER TABLE `test`.`test` ADD COLUMN d int(11) AS (1 + 1) STORED", - } - - var b []byte - bf := bytes.NewBuffer(b) - schemaName := model.NewCIStr("test") - ctx := &format.RestoreCtx{ - Flags: format.DefaultRestoreFlags, - In: bf, - } - - for _, sql := range sqls { - fmt.Println("\n###################################################") - fmt.Println("handle", sql) - stmts, warnings, err := parser.New().Parse(sql, "", "") - fmt.Println("result:", warnings, err) - - for _, stmt := range stmts { - switch v := stmt.(type) { - case *ast.CreateDatabaseStmt: - v.IfNotExists = true - case *ast.DropDatabaseStmt: - v.IfExists = true - case *ast.DropTableStmt: - v.IfExists = true - for _, t := range v.Tables { - if t.Schema.O == "" { - t.Schema = schemaName - } - } - case *ast.CreateTableStmt: - v.IfNotExists = true - if v.Table.Schema.O == "" { - v.Table.Schema = schemaName - } - - if v.ReferTable != nil && v.ReferTable.Schema.O == "" { - v.ReferTable.Schema = schemaName - } - case *ast.TruncateTableStmt: - if v.Table.Schema.O == "" { - v.Table.Schema = schemaName - } - case *ast.DropIndexStmt: - v.IfExists = true - if v.Table.Schema.O == "" { - v.Table.Schema = schemaName - } - case *ast.CreateIndexStmt: - if v.Table.Schema.O == "" { - v.Table.Schema = schemaName - } - case *ast.RenameTableStmt: - t2ts := v.TableToTables - for _, t2t := range t2ts { - if t2t.OldTable.Schema.O == "" { - t2t.OldTable.Schema = schemaName - } - if t2t.NewTable.Schema.O == "" { - t2t.NewTable.Schema = schemaName - } - - v.TableToTables = []*ast.TableToTable{t2t} - - bf.Reset() - stmt.Restore(ctx) - fmt.Println(bf.String()) - } - continue - case *ast.AlterTableStmt: - specs := v.Specs - - if v.Table.Schema.O == "" { - v.Table.Schema = schemaName - } - - for _, spec := range specs { - if spec.Tp == ast.AlterTableRenameTable { - if spec.NewTable.Schema.O == "" { - spec.NewTable.Schema = schemaName - } - } - - v.Specs = []*ast.AlterTableSpec{spec} - - bf.Reset() - stmt.Restore(ctx) - fmt.Println(bf.String()) - - if spec.Tp == ast.AlterTableRenameTable { - v.Table = spec.NewTable - } - } - - continue - } - - bf.Reset() - stmt.Restore(ctx) - fmt.Println(bf.String()) - } - } -} From 5f65a2617bd3353f2e6a48bece143bed78529503 Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Tue, 26 Feb 2019 18:29:37 +0800 Subject: [PATCH 18/20] fix test case --- syncer/ddl_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/syncer/ddl_test.go b/syncer/ddl_test.go index a805a1f075..8c0c0fb1a0 100644 --- a/syncer/ddl_test.go +++ b/syncer/ddl_test.go @@ -22,7 +22,6 @@ import ( parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/dm/pkg/utils" "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" ) @@ -356,16 +355,5 @@ func (s *testSyncerSuite) TestResolveGeneratedColumnSQL(c *C) { c.Assert(len(sqls), Equals, 1) getSQL := sqls[0] c.Assert(getSQL, Equals, tc.expected) - - ast2, err := parser.ParseOneStmt(getSQL, "", "") - c.Assert(err, IsNil) - - // compare parsed ast of the resoved SQL with parsed ast of the origin SQL. - // because text fields are not always same, and the difference of text - // makes no sense to the semantics, we just ignore checking it. - atStmt1 := ast1.(*ast.AlterTableStmt) - atStmt2 := ast2.(*ast.AlterTableStmt) - c.Assert(atStmt1.Table, DeepEquals, atStmt2.Table) - c.Assert(atStmt1.Specs, DeepEquals, atStmt2.Specs) } } From ec650ec19caeda190dbb043d6bdabb6b6fdc878f Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Thu, 7 Mar 2019 18:13:12 +0800 Subject: [PATCH 19/20] *: address comment --- pkg/parser/common.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/parser/common.go b/pkg/parser/common.go index 5ac550c399..f7b264f85e 100644 --- a/pkg/parser/common.go +++ b/pkg/parser/common.go @@ -32,7 +32,7 @@ var ( // ErrRenameMultipleTables is error that don't allow to rename multiple tables in one statement ErrRenameMultipleTables = errors.New("not allow operation: rename multiple tables in one statement") // ErrAlterMultipleTables is error that don't allow to alter multiple tables in one statement - ErrAlterMultipleTables = errors.New("not allow operation: rename multiple tables in one statement") + ErrAlterMultipleTables = errors.New("not allow operation: alter multiple tables in one statement") ) // Parse wraps parser.Parse(), makes `parser` suitable for dm @@ -40,11 +40,11 @@ func Parse(p *parser.Parser, sql, charset, collation string) (stmt []ast.StmtNod stmts, warnings, err := p.Parse(sql, charset, collation) for _, warning := range warnings { - log.Warnf("warning: parsing sql %s:%v", sql, warning) + log.Warnf("parsing sql %s:%v", sql, warning) } if err != nil { - log.Errorf("error: parsing sql %s:%v", sql, err) + log.Errorf("parsing sql %s:%v", sql, err) } return stmts, errors.Trace(err) @@ -86,7 +86,7 @@ func FetchDDLTableNames(schema string, stmt ast.StmtNode) ([]*filter.Table, erro case *ast.DropIndexStmt: res = append(res, genTableName(v.Table.Schema.O, v.Table.Name.O)) default: - return res, errors.Errorf("unkown type ddl %s", stmt) + return res, errors.Errorf("unknown type ddl %s", stmt) } for i := range res { @@ -179,9 +179,8 @@ func RenameDDLTable(stmt ast.StmtNode, targetTableNames []*filter.Table) (string // if fail to restore, it would not restore the value of `stmt` (it changes it's values if `stmt` is one of DropTableStmt, RenameTableStmt, AlterTableStmt) func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { var ( - b []byte schemaName = model.NewCIStr(schema) // fill schema name - bf = bytes.NewBuffer(b) + bf = new(bytes.Buffer) ctx = &format.RestoreCtx{ Flags: format.DefaultRestoreFlags, In: bf, @@ -261,6 +260,7 @@ func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { return sqls, nil case *ast.AlterTableStmt: specs := v.Specs + table := v.Table if v.Table.Schema.O == "" { v.Table.Schema = schemaName @@ -287,11 +287,12 @@ func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { } } v.Specs = specs + v.Table = table return sqls, nil default: - return nil, errors.Errorf("unkown type ddl %+v", stmt) + return nil, errors.Errorf("unknown type ddl %+v", stmt) } bf.Reset() From e2a3d91ac8e4fd7df0e66264b0f2a6de1bf3d2ea Mon Sep 17 00:00:00 2001 From: GregoryIan Date: Thu, 7 Mar 2019 19:11:39 +0800 Subject: [PATCH 20/20] *: restore ast while meeting any error in restore --- pkg/parser/common.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/parser/common.go b/pkg/parser/common.go index f7b264f85e..9c09b7ea6e 100644 --- a/pkg/parser/common.go +++ b/pkg/parser/common.go @@ -205,6 +205,7 @@ func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { bf.Reset() err = stmt.Restore(ctx) if err != nil { + v.Tables = tables return nil, errors.Annotate(err, "restore ast node") } @@ -250,6 +251,7 @@ func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { bf.Reset() err = stmt.Restore(ctx) if err != nil { + v.TableToTables = t2ts return nil, errors.Annotate(err, "restore ast node") } @@ -278,6 +280,8 @@ func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { bf.Reset() err = stmt.Restore(ctx) if err != nil { + v.Specs = specs + v.Table = table return nil, errors.Annotate(err, "restore ast node") } sqls = append(sqls, bf.String()) @@ -290,7 +294,6 @@ func SplitDDL(stmt ast.StmtNode, schema string) (sqls []string, err error) { v.Table = table return sqls, nil - default: return nil, errors.Errorf("unknown type ddl %+v", stmt) }