Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dep(*): update tidb to include DM SchemaTracker fix (#11410) #11699

Closed
wants to merge 1 commit into from
Closed

dep(*): update tidb to include DM SchemaTracker fix (#11410) #11699

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions dm/pkg/schema/tracker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,12 +216,12 @@ func (tr *Tracker) GetTableInfo(table *filter.Table) (*model.TableInfo, error) {
if tr.closed.Load() {
return nil, dmterror.ErrSchemaTrackerIsClosed.New("fail to get table info")
}
return tr.upstreamTracker.TableByName(model.NewCIStr(table.Schema), model.NewCIStr(table.Name))
return tr.upstreamTracker.TableByName(context.Background(), model.NewCIStr(table.Schema), model.NewCIStr(table.Name))
}

// GetCreateTable returns the `CREATE TABLE` statement of the table.
func (tr *Tracker) GetCreateTable(ctx context.Context, table *filter.Table) (string, error) {
tableInfo, err := tr.upstreamTracker.TableByName(model.NewCIStr(table.Schema), model.NewCIStr(table.Name))
tableInfo, err := tr.upstreamTracker.TableByName(ctx, model.NewCIStr(table.Schema), model.NewCIStr(table.Name))
if err != nil {
return "", err
}
Expand Down Expand Up @@ -252,7 +252,7 @@ func (tr *Tracker) ListSchemaTables(schema string) ([]string, error) {
// TODO: move out of this package!
func (tr *Tracker) GetSingleColumnIndices(db, tbl, col string) ([]*model.IndexInfo, error) {
col = strings.ToLower(col)
t, err := tr.upstreamTracker.TableByName(model.NewCIStr(db), model.NewCIStr(tbl))
t, err := tr.upstreamTracker.TableByName(context.Background(), model.NewCIStr(db), model.NewCIStr(tbl))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -334,13 +334,19 @@ func (tr *Tracker) CreateTableIfNotExists(table *filter.Table, ti *model.TableIn
tableName := model.NewCIStr(table.Name)
ti = cloneTableInfo(ti)
ti.Name = tableName
<<<<<<< HEAD
return tr.upstreamTracker.CreateTableWithInfo(tr.se, schemaName, ti, ddl.OnExistIgnore)
=======
return tr.upstreamTracker.CreateTableWithInfo(tr.se, schemaName, ti, nil, ddl.WithOnExist(ddl.OnExistIgnore))
>>>>>>> 2fcbb315f0 (dep(*): update tidb to include DM SchemaTracker fix (#11410))
}

// SplitBatchCreateTableAndHandle will split the batch if it exceeds the kv entry size limit.
func (tr *Tracker) SplitBatchCreateTableAndHandle(schema model.CIStr, info []*model.TableInfo, l int, r int) error {
var err error
if err = tr.upstreamTracker.BatchCreateTableWithInfo(tr.se, schema, info[l:r], ddl.OnExistIgnore); kv.ErrEntryTooLarge.Equal(err) {
if err = tr.upstreamTracker.BatchCreateTableWithInfo(
tr.se, schema, info[l:r], ddl.WithOnExist(ddl.OnExistIgnore),
); kv.ErrEntryTooLarge.Equal(err) {
if r-l == 1 {
return err
}
Expand Down
1 change: 0 additions & 1 deletion dm/syncer/expr_filter_group.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,6 @@ func (g *ExprFilterGroup) ResetExprs(table *filter.Table) {
// SkipDMLByExpression returns true when given row matches the expr, which means this row should be skipped.
func SkipDMLByExpression(ctx sessionctx.Context, row []interface{}, expr expression.Expression, upstreamCols []*model.ColumnInfo) (bool, error) {
// TODO: add MetricsProxies
log.L().Debug("will evaluate the expression", zap.Stringer("expression", expr), zap.Any("raw row", row))
data, err := utils.AdjustBinaryProtocolForDatum(ctx, row, upstreamCols)
if err != nil {
return false, err
Expand Down
9 changes: 8 additions & 1 deletion dm/syncer/expr_filter_group_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,16 @@ import (
"context"
"testing"

<<<<<<< HEAD
ddl2 "github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/util/filter"
=======
ddl2 "github.com/pingcap/tidb/pkg/ddl"
context2 "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/util/filter"
>>>>>>> 2fcbb315f0 (dep(*): update tidb to include DM SchemaTracker fix (#11410))
"github.com/pingcap/tiflow/dm/config"
tcontext "github.com/pingcap/tiflow/dm/pkg/context"
"github.com/pingcap/tiflow/dm/pkg/log"
Expand Down Expand Up @@ -439,7 +446,7 @@ create table t (
require.NoError(t, err)
require.Len(t, exprs, 1)
expr := exprs[0]
require.Equal(t, "0", expr.String())
require.Equal(t, "0", expr.StringWithCtx(context2.EmptyParamValues))

// skip nothing
skip, err := SkipDMLByExpression(sessCtx, []interface{}{0}, expr, ti.Columns)
Expand Down
146 changes: 146 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ require (
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
github.com/pierrec/lz4/v4 v4.1.17
github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0
<<<<<<< HEAD
github.com/pingcap/errors v0.11.5-0.20231212100244-799fae176cfb
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c
github.com/pingcap/kvproto v0.0.0-20231011074246-fa00d2b03372
Expand All @@ -69,6 +70,17 @@ require (
github.com/pingcap/tidb/parser v0.0.0-20240226095130-a26af5fcf857
github.com/prometheus/client_golang v1.15.1
github.com/prometheus/client_model v0.3.0
=======
github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
github.com/pingcap/kvproto v0.0.0-20240620063548-118a4cab53e4
github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d
github.com/pingcap/tidb v1.1.0-beta.0.20240722024203-504960d51b2a
github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7
github.com/pingcap/tidb/pkg/parser v0.0.0-20240722015532-8edd4ed54376
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/client_model v0.6.1
>>>>>>> 2fcbb315f0 (dep(*): update tidb to include DM SchemaTracker fix (#11410))
github.com/r3labs/diff v1.1.0
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
github.com/robfig/cron v1.2.0
Expand All @@ -83,9 +95,16 @@ require (
github.com/swaggo/gin-swagger v1.2.0
github.com/swaggo/swag v1.8.3
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954
<<<<<<< HEAD
github.com/tikv/client-go/v2 v2.0.8-0.20231211100325-d44bb7f9cb9e
github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132
github.com/tikv/pd/client v0.0.0-20231211083919-fe6fd1721aa6
=======
github.com/thanhpk/randstr v1.0.6
github.com/tikv/client-go/v2 v2.0.8-0.20240703095801-d73cc1ed6503
github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b
github.com/tikv/pd/client v0.0.0-20240717053728-5ec6af403019
>>>>>>> 2fcbb315f0 (dep(*): update tidb to include DM SchemaTracker fix (#11410))
github.com/tinylib/msgp v1.1.6
github.com/uber-go/atomic v1.4.0
github.com/vmihailenco/msgpack/v5 v5.3.5
Expand All @@ -102,6 +121,7 @@ require (
go.uber.org/goleak v1.2.1
go.uber.org/multierr v1.11.0
go.uber.org/ratelimit v0.2.0
<<<<<<< HEAD
go.uber.org/zap v1.24.0
golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e
golang.org/x/net v0.17.0
Expand All @@ -113,6 +133,20 @@ require (
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1
google.golang.org/grpc v1.54.0
google.golang.org/protobuf v1.30.0
=======
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8
golang.org/x/net v0.27.0
golang.org/x/oauth2 v0.21.0
golang.org/x/sync v0.7.0
golang.org/x/sys v0.22.0
golang.org/x/text v0.16.0
golang.org/x/time v0.5.0
google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda
google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291
google.golang.org/grpc v1.64.0
google.golang.org/protobuf v1.34.2
>>>>>>> 2fcbb315f0 (dep(*): update tidb to include DM SchemaTracker fix (#11410))
gopkg.in/yaml.v2 v2.4.0
gorm.io/driver/mysql v1.3.3
gorm.io/gorm v1.23.8
Expand All @@ -128,7 +162,97 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
<<<<<<< HEAD
github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 // indirect
=======
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/apache/arrow/go/v12 v12.0.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/bytedance/sonic v1.9.1 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/dolthub/maphash v0.1.0 // indirect
github.com/dolthub/swiss v0.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
github.com/go-ldap/ldap/v3 v3.4.4 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-resty/resty/v2 v2.11.0 // indirect
github.com/goccy/go-reflect v1.2.0 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/google/flatbuffers v2.0.8+incompatible // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/influxdata/tdigest v0.0.1 // indirect
github.com/jellydator/ttlcache/v3 v3.0.1 // indirect
github.com/jfcg/sixb v1.3.8 // indirect
github.com/jfcg/sorty/v2 v2.1.0 // indirect
github.com/joomcode/errorx v1.0.1 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/ks3sdklib/aws-sdk-go v1.2.9 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/otiai10/copy v1.2.0 // indirect
github.com/qri-io/jsonpointer v0.1.1 // indirect
github.com/qri-io/jsonschema v0.2.1 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/tidwall/btree v1.7.0 // indirect
github.com/tidwall/buntdb v1.3.0 // indirect
github.com/tidwall/gjson v1.14.3 // indirect
github.com/tidwall/grect v0.1.4 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/rtred v0.1.2 // indirect
github.com/tidwall/tinyqueue v0.1.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
k8s.io/api v0.28.6 // indirect
k8s.io/apimachinery v0.28.6 // indirect
k8s.io/klog/v2 v2.120.1 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
)

require (
cloud.google.com/go v0.112.2 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/iam v1.1.7 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.1 // indirect
github.com/AthenZ/athenz v1.10.39 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect
github.com/DataDog/zstd v1.5.5 // indirect
>>>>>>> 2fcbb315f0 (dep(*): update tidb to include DM SchemaTracker fix (#11410))
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
Expand Down Expand Up @@ -245,11 +369,19 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
<<<<<<< HEAD
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
=======
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
>>>>>>> 2fcbb315f0 (dep(*): update tidb to include DM SchemaTracker fix (#11410))
github.com/rs/cors v1.7.0 // indirect
github.com/sasha-s/go-deadlock v0.2.0 // indirect
github.com/shoenig/go-m1cpu v0.1.5 // indirect
Expand Down Expand Up @@ -282,6 +414,7 @@ require (
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/client/v2 v2.305.4 // indirect
go.opencensus.io v0.24.0 // indirect
<<<<<<< HEAD
go.opentelemetry.io/contrib v0.20.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect
go.opentelemetry.io/otel v0.20.0 // indirect
Expand All @@ -299,6 +432,19 @@ require (
google.golang.org/api v0.114.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
=======
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/sdk v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.opentelemetry.io/proto/otlp v1.1.0 // indirect
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/term v0.22.0
golang.org/x/tools v0.22.0 // indirect
google.golang.org/api v0.170.0 // indirect
>>>>>>> 2fcbb315f0 (dep(*): update tidb to include DM SchemaTracker fix (#11410))
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
Loading
Loading