Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mounter(ticdc): fix mounter add default value type unsupported #3846

Merged
merged 7 commits into from
Dec 13, 2021
Merged

mounter(ticdc): fix mounter add default value type unsupported #3846

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b55bb25
mounter(ticdc): fix mounter add default value type unsupported
Dec 10, 2021
aa1e847
test(mounter(ticdc)): fix unit test panic for enum
Dec 13, 2021
80f59cb
comment(mounter(ticdc)): add some return value comment for mounter
Dec 13, 2021
ae6e73d
test(mounter(ticdc)): fix default value size unmatch error in uts
Dec 13, 2021
fd172de
mounter(ticdc): change comment
Dec 13, 2021
4268a1f
test(mounter(ticdc)): close size check temporary
Dec 13, 2021
f147dac
chore(mounter(ticdc)): fix unused lint
Dec 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 41 additions & 38 deletions cdc/entry/mounter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,25 +313,24 @@ func datum2Column(tableInfo *model.TableInfo, datums map[int64]types.Datum, fill
colName := colInfo.Name.O
colDatums, exist := datums[colInfo.ID]
var colValue interface{}
if !exist && !fillWithDefaultValue {
continue
}
var err error
var warn string
var size int
if exist {
var err error
var warn string
var size int
colValue, size, warn, err = formatColVal(colDatums, colInfo.Tp)
if err != nil {
return nil, errors.Trace(err)
}
if warn != "" {
log.Warn(warn, zap.String("table", tableInfo.TableName.String()), zap.String("column", colInfo.Name.String()))
}
colSize += size
} else if fillWithDefaultValue {
var size int
colValue, size = getDefaultOrZeroValue(colInfo)
colSize += size
} else {
continue
colValue, size, warn, err = getDefaultOrZeroValue(colInfo)
}
if err != nil {
return nil, errors.Trace(err)
}
if warn != "" {
log.Warn(warn, zap.String("table", tableInfo.TableName.String()), zap.String("column", colInfo.Name.String()))
}
colSize += size
cols[tableInfo.RowColumnsOffset[colInfo.ID]] = &model.Column{
Name: colName,
Type: colInfo.Tp,
Expand Down Expand Up @@ -437,6 +436,7 @@ func sizeOfBytes(b []byte) int {
return len(b) + sizeOfEmptyBytes
}

// formatColVal return interface{} need to meet the same requirement as getDefaultOrZeroValue
func formatColVal(datum types.Datum, tp byte) (
value interface{}, size int, warn string, err error,
) {
Expand Down Expand Up @@ -490,36 +490,39 @@ func formatColVal(datum types.Datum, tp byte) (
const sizeOfV = unsafe.Sizeof(v)
return v, int(sizeOfV), warn, nil
default:
// NOTICE: GetValue() may return some types that go sql not support, which will cause sink DML fail
maxshuang marked this conversation as resolved.
Show resolved Hide resolved
// Make specified convert upper if you need
// Go sql support type ref to: https://github.com/golang/go/blob/go1.17.4/src/database/sql/driver/types.go#L236
return datum.GetValue(), sizeOfDatum(datum), "", nil
}
}

func getDefaultOrZeroValue(col *timodel.ColumnInfo) (interface{}, int) {
// see https://github.com/pingcap/tidb/issues/9304
// must use null if TiDB not write the column value when default value is null
// and the value is null
// getDefaultOrZeroValue return interface{} need to meet to require type in
// https://github.com/golang/go/blob/go1.17.4/src/database/sql/driver/types.go#L236
// Supported type is: nil, basic type(Int, Int8,..., Float32, Float64, String), Slice(uint8), other types not support
func getDefaultOrZeroValue(col *timodel.ColumnInfo) (interface{}, int, string, error) {
maxshuang marked this conversation as resolved.
Show resolved Hide resolved
var d types.Datum
if !mysql.HasNotNullFlag(col.Flag) {
d := types.NewDatum(nil)
const size = unsafe.Sizeof(d.GetValue())
return d.GetValue(), int(size)
}

if col.GetDefaultValue() != nil {
d := types.NewDatum(col.GetDefaultValue())
return d.GetValue(), sizeOfDatum(d)
}
switch col.Tp {
case mysql.TypeEnum:
// For enum type, if no default value and not null is set,
// the default value is the first element of the enum list
d := types.NewDatum(col.FieldType.Elems[0])
return d.GetValue(), sizeOfDatum(d)
case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar:
return emptyBytes, sizeOfEmptyBytes
// see https://github.com/pingcap/tidb/issues/9304
// must use null if TiDB not write the column value when default value is null
// and the value is null
d = types.NewDatum(nil)
} else if col.GetDefaultValue() != nil {
d = types.NewDatum(col.GetDefaultValue())
} else {
switch col.Tp {
case mysql.TypeEnum:
// For enum type, if no default value and not null is set,
// the default value is the first element of the enum list
d = types.NewDatum(col.FieldType.Elems[0])
case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar:
return emptyBytes, sizeOfEmptyBytes, "", nil
default:
d = table.GetZeroValue(col)
}
}

d := table.GetZeroValue(col)
return d.GetValue(), sizeOfDatum(d)
return formatColVal(d, col.Tp)
}

// DecodeTableID decodes the raw key to a table ID
Expand Down
Loading