Skip to content
This repository has been archived by the owner on Nov 24, 2023. It is now read-only.

Commit

Permalink
syncer: use strings.Builder to replace fmt.Sprintf; 470 -> 457
Browse files Browse the repository at this point in the history
  • Loading branch information
@csuzhangxc
csuzhangxc committed Apr 11, 2020
1 parent 26b7e39 commit 1b68cd3
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 115 deletions.
188 changes: 99 additions & 89 deletions syncer/dml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
package syncer

import (
"bytes"
"encoding/binary"
"fmt"
"io"
"strconv"
"strings"

Expand Down Expand Up @@ -102,42 +102,35 @@ func extractValueFromData(data []interface{}, columns []*column) []interface{} {

func genInsertSQLs(param *genDMLParam) ([]string, [][]string, [][]interface{}, error) {
var (
schema = param.schema
table = param.table
fullname = dbutil.TableName(param.schema, param.table)
dataSeq = param.data
originalDataSeq = param.originalData
columns = param.columns
originalColumns = param.originalColumns
originalIndexColumns = param.originalIndexColumns
sqls = make([]string, 0, len(dataSeq))
keys = make([][]string, 0, len(dataSeq))
values = make([][]interface{}, 0, len(dataSeq))
)
sqls := make([]string, 0, len(dataSeq))
keys := make([][]string, 0, len(dataSeq))
values := make([][]interface{}, 0, len(dataSeq))
columnList := genColumnList(columns)
columnPlaceholders := genColumnPlaceholders(len(columns))

var insertOrReplace = "INSERT INTO"
if param.safeMode {
insertOrReplace = "REPLACE INTO"
}
sql := genInsertReplace(insertOrReplace, fullname, columns)

for dataIdx, data := range dataSeq {
if len(data) != len(columns) {
return nil, nil, nil, terror.ErrSyncerUnitDMLColumnNotMatch.Generate(len(columns), len(data))
}

value := extractValueFromData(data, columns)
originalData := originalDataSeq[dataIdx]
var originalValue []interface{}
if len(columns) == len(originalColumns) {
originalValue = value
} else {
originalValue = extractValueFromData(originalData, originalColumns)
var originalValue = value
if len(columns) != len(originalColumns) {
originalValue = extractValueFromData(originalDataSeq[dataIdx], originalColumns)
}

var insertOrReplace string
if param.safeMode {
insertOrReplace = "REPLACE"
} else {
insertOrReplace = "INSERT"
}

sql := fmt.Sprintf("%s INTO `%s`.`%s` (%s) VALUES (%s);", insertOrReplace, schema, table, columnList, columnPlaceholders)
ks := genMultipleKeys(originalColumns, originalValue, originalIndexColumns, dbutil.TableName(schema, table))
ks := genMultipleKeys(originalColumns, originalValue, originalIndexColumns, fullname)
sqls = append(sqls, sql)
values = append(values, value)
keys = append(keys, ks)
Expand All @@ -148,21 +141,22 @@ func genInsertSQLs(param *genDMLParam) ([]string, [][]string, [][]interface{}, e

func genUpdateSQLs(param *genDMLParam) ([]string, [][]string, [][]interface{}, error) {
var (
schema = param.schema
table = param.table
safeMode = param.safeMode
fullname = dbutil.TableName(param.schema, param.table)
data = param.data
originalData = param.originalData
columns = param.columns
originalColumns = param.originalColumns
originalIndexColumns = param.originalIndexColumns
defaultIndexColumns = findFitIndex(originalIndexColumns)
replaceSQL string // `REPLACE INTO` SQL
sqls = make([]string, 0, len(data)/2)
keys = make([][]string, 0, len(data)/2)
values = make([][]interface{}, 0, len(data)/2)
)
sqls := make([]string, 0, len(data)/2)
keys := make([][]string, 0, len(data)/2)
values := make([][]interface{}, 0, len(data)/2)
columnList := genColumnList(columns)
columnPlaceholders := genColumnPlaceholders(len(columns))
defaultIndexColumns := findFitIndex(originalIndexColumns)

if param.safeMode {
replaceSQL = genInsertReplace("REPLACE INTO", fullname, columns)
}

for i := 0; i < len(data); i += 2 {
oldData := data[i]
Expand Down Expand Up @@ -194,23 +188,23 @@ func genUpdateSQLs(param *genDMLParam) ([]string, [][]string, [][]interface{}, e
defaultIndexColumns = getAvailableIndexColumn(originalIndexColumns, oriOldValues)
}

ks := genMultipleKeys(originalColumns, oriOldValues, originalIndexColumns, dbutil.TableName(schema, table))
ks = append(ks, genMultipleKeys(originalColumns, oriChangedValues, originalIndexColumns, dbutil.TableName(schema, table))...)
ks := genMultipleKeys(originalColumns, oriOldValues, originalIndexColumns, fullname)
ks = append(ks, genMultipleKeys(originalColumns, oriChangedValues, originalIndexColumns, fullname)...)

if safeMode {
if param.safeMode {
// generate delete sql from old data
sql, value := genDeleteSQL(schema, table, oriOldValues, originalColumns, defaultIndexColumns)
sql, value := genDeleteSQL(fullname, oriOldValues, originalColumns, defaultIndexColumns)
sqls = append(sqls, sql)
values = append(values, value)
keys = append(keys, ks)
// generate replace sql from new data
sql = fmt.Sprintf("REPLACE INTO `%s`.`%s` (%s) VALUES (%s);", schema, table, columnList, columnPlaceholders)
sqls = append(sqls, sql)
sqls = append(sqls, replaceSQL)
values = append(values, changedValues)
keys = append(keys, ks)
continue
}

// NOTE: move these variables outer of `for` if needed (to reuse).
updateColumns := make([]*column, 0, len(defaultIndexColumns))
updateValues := make([]interface{}, 0, len(defaultIndexColumns))
for j := range oldValues {
Expand All @@ -224,18 +218,16 @@ func genUpdateSQLs(param *genDMLParam) ([]string, [][]string, [][]interface{}, e
}

value := make([]interface{}, 0, len(oldData))
kvs := genKVs(updateColumns)
value = append(value, updateValues...)

whereColumns, whereValues := originalColumns, oriOldValues
if len(defaultIndexColumns) > 0 {
whereColumns, whereValues = getColumnData(originalColumns, defaultIndexColumns, oriOldValues)
}

where := genWhere(whereColumns, whereValues)
value = append(value, whereValues...)

sql := fmt.Sprintf("UPDATE `%s`.`%s` SET %s WHERE %s LIMIT 1;", schema, table, kvs, where)
sql := genUpdateSQL(fullname, updateColumns, whereColumns, whereValues)
sqls = append(sqls, sql)
values = append(values, value)
keys = append(keys, ks)
Expand All @@ -246,16 +238,15 @@ func genUpdateSQLs(param *genDMLParam) ([]string, [][]string, [][]interface{}, e

func genDeleteSQLs(param *genDMLParam) ([]string, [][]string, [][]interface{}, error) {
var (
schema = param.schema
table = param.table
dataSeq = param.originalData
columns = param.originalColumns
indexColumns = param.originalIndexColumns
fullname = dbutil.TableName(param.schema, param.table)
dataSeq = param.originalData
columns = param.originalColumns
indexColumns = param.originalIndexColumns
defaultIndexColumns = findFitIndex(indexColumns)
sqls = make([]string, 0, len(dataSeq))
keys = make([][]string, 0, len(dataSeq))
values = make([][]interface{}, 0, len(dataSeq))
)
sqls := make([]string, 0, len(dataSeq))
keys := make([][]string, 0, len(dataSeq))
values := make([][]interface{}, 0, len(dataSeq))
defaultIndexColumns := findFitIndex(indexColumns)

for _, data := range dataSeq {
if len(data) != len(columns) {
Expand All @@ -267,9 +258,9 @@ func genDeleteSQLs(param *genDMLParam) ([]string, [][]string, [][]interface{}, e
if len(defaultIndexColumns) == 0 {
defaultIndexColumns = getAvailableIndexColumn(indexColumns, value)
}
ks := genMultipleKeys(columns, value, indexColumns, dbutil.TableName(schema, table))
ks := genMultipleKeys(columns, value, indexColumns, fullname)

sql, value := genDeleteSQL(schema, table, value, columns, defaultIndexColumns)
sql, value := genDeleteSQL(fullname, value, columns, defaultIndexColumns)
sqls = append(sqls, sql)
values = append(values, value)
keys = append(keys, ks)
Expand All @@ -278,37 +269,72 @@ func genDeleteSQLs(param *genDMLParam) ([]string, [][]string, [][]interface{}, e
return sqls, keys, values, nil
}

func genDeleteSQL(schema string, table string, value []interface{}, columns []*column, indexColumns []*column) (string, []interface{}) {
whereColumns, whereValues := columns, value
if len(indexColumns) > 0 {
whereColumns, whereValues = getColumnData(columns, indexColumns, value)
}

where := genWhere(whereColumns, whereValues)
sql := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE %s LIMIT 1;", schema, table, where)

return sql, whereValues
}

func genColumnList(columns []*column) string {
// genInsertReplace generates a DML for `INSERT INTO` or `REPLCATE INTO`.
// the returned SQL with placeholders for `VALUES`.
func genInsertReplace(op, table string, columns []*column) string {
// NOTE: use sync.Pool to hold the builder if needed later.
var buf strings.Builder
buf.Grow(256)
buf.WriteString(op)
buf.WriteString(" " + table + " ")
for i, column := range columns {
if i != len(columns)-1 {
buf.WriteString("`" + column.name + "`,")
} else {
buf.WriteString("`" + column.name + "`")
}
}
buf.WriteString(" VALUES ")

// placeholders
for i := range columns {
if i != len(columns)-1 {
buf.WriteString("?,")
} else {
buf.WriteString("?")
}
}
return buf.String()
}

func genColumnPlaceholders(length int) string {
values := make([]string, length)
for i := 0; i < length; i++ {
values[i] = "?"
// genUpdateSQL generates a `UPDATE` SQL with `SET` and `WHERE`.
func genUpdateSQL(table string, updateColumns, whereColumns []*column, whereValues []interface{}) string {
var buf strings.Builder
buf.Grow(2048)
buf.WriteString("UPDATE ")
buf.WriteString(table)
buf.WriteString(" SET ")

for i, column := range updateColumns {
if i == len(updateColumns)-1 {
fmt.Fprintf(&buf, "`%s` = ?", column.name) // TODO: update `tidb-tools` to use `dbutil.ColumnName`.
} else {
fmt.Fprintf(&buf, "`%s` = ?, ", column.name)
}
}
return strings.Join(values, ",")

buf.WriteString(" WHERE ")
genWhere(&buf, whereColumns, whereValues)
buf.WriteString(" LIMIT 1")
return buf.String()
}

// genDeleteSQL generates a `DELETE FROM` SQL with `WHERE`.
func genDeleteSQL(table string, value []interface{}, columns []*column, indexColumns []*column) (string, []interface{}) {
whereColumns, whereValues := columns, value
if len(indexColumns) > 0 {
whereColumns, whereValues = getColumnData(columns, indexColumns, value)
}

var buf strings.Builder
buf.Grow(1024)
buf.WriteString("DELETE FROM ")
buf.WriteString(table)
buf.WriteString(" WHERE ")
genWhere(&buf, whereColumns, whereValues)
buf.WriteString(" LIMIT 1")

return buf.String(), whereValues
}

func castUnsigned(data interface{}, unsigned bool, tp string) interface{} {
Expand Down Expand Up @@ -490,35 +516,19 @@ func getColumnData(columns []*column, indexColumns []*column, data []interface{}
return cols, values
}

func genWhere(columns []*column, data []interface{}) string {
var kvs bytes.Buffer
func genWhere(w io.Writer, columns []*column, data []interface{}) {
for i := range columns {
kvSplit := "="
if data[i] == nil {
kvSplit = "IS"
}

if i == len(columns)-1 {
fmt.Fprintf(&kvs, "`%s` %s ?", columns[i].name, kvSplit)
} else {
fmt.Fprintf(&kvs, "`%s` %s ? AND ", columns[i].name, kvSplit)
}
}

return kvs.String()
}

func genKVs(columns []*column) string {
var kvs bytes.Buffer
for i := range columns {
if i == len(columns)-1 {
fmt.Fprintf(&kvs, "`%s` = ?", columns[i].name)
fmt.Fprintf(w, "`%s` %s ?", columns[i].name, kvSplit)
} else {
fmt.Fprintf(&kvs, "`%s` = ?, ", columns[i].name)
fmt.Fprintf(w, "`%s` %s ? AND ", columns[i].name, kvSplit)
}
}

return kvs.String()
}

func (s *Syncer) mappingDML(schema, table string, columns []string, data [][]interface{}) ([][]interface{}, error) {
Expand Down
26 changes: 0 additions & 26 deletions syncer/dml_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,32 +45,6 @@ func (s *testSyncerSuite) TestCastUnsigned(c *C) {
}
}

func (s *testSyncerSuite) TestGenColumnPlaceholders(c *C) {
placeholderStr := genColumnPlaceholders(1)
c.Assert(placeholderStr, Equals, "?")

placeholderStr = genColumnPlaceholders(3)
c.Assert(placeholderStr, Equals, "?,?,?")
}

func (s *testSyncerSuite) TestGenColumnList(c *C) {
columns := []*column{
{
name: "a",
}, {
name: "b",
}, {
name: "c",
},
}

columnList := genColumnList(columns[:1])
c.Assert(columnList, Equals, "`a`")

columnList = genColumnList(columns)
c.Assert(columnList, Equals, "`a`,`b`,`c`")
}

func (s *testSyncerSuite) TestFindFitIndex(c *C) {
pkColumns := []*column{
{
Expand Down

0 comments on commit 1b68cd3

Please sign in to comment.