Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

expression: add builtin function tidb_decode_key() #12193

Merged
merged 21 commits into from
Sep 24, 2019
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions expression/builtin.go
Original file line number Diff line number Diff line change
Expand Up @@ -719,4 +719,7 @@ var funcs = map[string]functionClass{
ast.JSONDepth: &jsonDepthFunctionClass{baseFunctionClass{ast.JSONDepth, 1, 1}},
ast.JSONKeys: &jsonKeysFunctionClass{baseFunctionClass{ast.JSONKeys, 1, 2}},
ast.JSONLength: &jsonLengthFunctionClass{baseFunctionClass{ast.JSONLength, 1, 2}},

// TiDB internal function.
ast.TiDBDecodeKey: &tidbDecodeKeyFunctionClass{baseFunctionClass{ast.TiDBDecodeKey, 1, 1}},
zz-jason marked this conversation as resolved.
Show resolved Hide resolved
}
67 changes: 67 additions & 0 deletions expression/builtin_info.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,18 @@
package expression

import (
"encoding/hex"
"fmt"
"sort"
"strconv"

"github.com/pingcap/errors"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/printer"
)

Expand All @@ -44,6 +49,7 @@ var (
_ functionClass = &rowCountFunctionClass{}
_ functionClass = &tidbVersionFunctionClass{}
_ functionClass = &tidbIsDDLOwnerFunctionClass{}
_ functionClass = &tidbDecodeKeyFunctionClass{}
)

var (
Expand All @@ -57,6 +63,7 @@ var (
_ builtinFunc = &builtinVersionSig{}
_ builtinFunc = &builtinTiDBVersionSig{}
_ builtinFunc = &builtinRowCountSig{}
_ builtinFunc = &builtinTiDBDecodeKeySig{}
)

type databaseFunctionClass struct {
Expand Down Expand Up @@ -589,3 +596,63 @@ func (b *builtinRowCountSig) evalInt(_ chunk.Row) (res int64, isNull bool, err e
res = int64(b.ctx.GetSessionVars().StmtCtx.PrevAffectedRows)
return res, false, nil
}

type tidbDecodeKeyFunctionClass struct {
baseFunctionClass
}

func (c *tidbDecodeKeyFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) {
if err := c.verifyArgs(args); err != nil {
return nil, err
}
bf := newBaseBuiltinFuncWithTp(ctx, args, types.ETString, types.ETString)
sig := &builtinTiDBDecodeKeySig{bf}
return sig, nil
}

type builtinTiDBDecodeKeySig struct {
baseBuiltinFunc
}

func (b *builtinTiDBDecodeKeySig) Clone() builtinFunc {
newSig := &builtinTiDBDecodeKeySig{}
newSig.cloneFrom(&b.baseBuiltinFunc)
return newSig
}

// evalInt evals a builtinTiDBIsDDLOwnerSig.
func (b *builtinTiDBDecodeKeySig) evalString(row chunk.Row) (string, bool, error) {
s, isNull, err := b.args[0].EvalString(b.ctx, row)
if isNull || err != nil {
return "", isNull, err
}
return decodeKey(b.ctx, s), false, nil
}

func decodeKey(ctx sessionctx.Context, s string) string {
key, err := hex.DecodeString(s)
if err != nil {
ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("invalid record/index key: %v", key))
return s
}
// Auto decode byte if needed.
_, bs, err := codec.DecodeBytes([]byte(key), nil)
if err == nil {
key = bs
}
// Try to decode it as a record key.
tableID, handle, err := tablecodec.DecodeRecordKey(key)
if err == nil {
return "tableID=" + strconv.FormatInt(tableID, 10) + ", _tidb_rowid=" + strconv.FormatInt(handle, 10)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why we don't return json object?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great idea.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will change this tomorrow.

}
// Try decode as table index key.
tableID, indexID, indexValues, err := tablecodec.DecodeIndexKeyPrefix(key)
if err == nil {
idxValueStr := fmt.Sprintf("%X", indexValues)
return "tableID=" + strconv.FormatInt(tableID, 10) + ", indexID=" + strconv.FormatInt(indexID, 10) + ", indexValues=" + idxValueStr
}

// TODO: try to decode other type key.
ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("the key is a invalid record/index key"))
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
return s
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
}
17 changes: 17 additions & 0 deletions expression/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4123,6 +4123,23 @@ func (s *testIntegrationSuite) testTiDBIsOwnerFunc(c *C) {
result.Check(testkit.Rows(fmt.Sprintf("%v", ret)))
}

func (s *testIntegrationSuite) TestTiDBInternalFunc(c *C) {
tk := testkit.NewTestKit(c, s.store)
defer s.cleanEnv(c)
result := tk.MustQuery("select tidb_decode_key( '74800000000000002B5F72800000000000A5D3' )")
result.Check(testkit.Rows("tableID=43, _tidb_rowid=42451"))

result = tk.MustQuery("select tidb_decode_key( '74800000000000019B5F698000000000000001015257303100000000FB013736383232313130FF3900000000000000F8010000000000000000F7' )")
result.Check(testkit.Rows("tableID=411, indexID=1, indexValues=015257303100000000FB013736383232313130FF3900000000000000F8010000000000000000F7"))

// Test invalid record/index key.
result = tk.MustQuery("select tidb_decode_key( '7480000000000000FF2E5F728000000011FFE1A3000000000000' )")
result.Check(testkit.Rows("7480000000000000FF2E5F728000000011FFE1A3000000000000"))
warns := tk.Se.GetSessionVars().StmtCtx.GetWarnings()
c.Assert(warns, HasLen, 1)
c.Assert(warns[0].Err.Error(), Equals, "the key is a invalid record/index key")
}

func newStoreWithBootstrap() (kv.Storage, *domain.Domain, error) {
store, err := mockstore.NewMockTikvStore()
if err != nil {
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,5 @@ require (
sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4
sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
)

replace github.com/pingcap/parser => github.com/crazycs520/parser v0.0.0-20190916053053-b800dfc88b6d
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/crazycs520/parser v0.0.0-20190916053053-b800dfc88b6d h1:UVtq5K54gpeYI5I8w4r/YYBg9XywJDZN87WEzasYAzU=
github.com/crazycs520/parser v0.0.0-20190916053053-b800dfc88b6d/go.mod h1:xLjI+gnWYexq011WPMEvCNS8rFM9qe1vdojIEzSKPuc=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE=
Expand Down Expand Up @@ -166,8 +168,6 @@ github.com/pingcap/kvproto v0.0.0-20190904075355-9a1bd6a31da2/go.mod h1:QMdbTAXC
github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w=
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+8794bFHY1bPrmu6bTalpssCrGg=
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw=
github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e h1:QeD1wC7bGElAhufSHH4JcIbs1cVdxnGWD3n3gcE5qeY=
github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY=
github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs=
github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU=
Expand Down
22 changes: 17 additions & 5 deletions tablecodec/tablecodec.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,14 +127,10 @@ func DecodeRecordKey(key kv.Key) (tableID int64, handle int64, err error) {
func DecodeIndexKey(key kv.Key) (tableID int64, indexID int64, indexValues []string, err error) {
k := key

tableID, indexID, isRecord, err := DecodeKeyHead(key)
tableID, indexID, key, err = DecodeIndexKeyPrefix(key)
if err != nil {
return 0, 0, nil, errors.Trace(err)
}
if isRecord {
return 0, 0, nil, errInvalidIndexKey.GenWithStack("invalid index key - %q", k)
}
key = key[prefixLen+idLen:]

for len(key) > 0 {
// FIXME: Without the schema information, we can only decode the raw kind of
Expand All @@ -153,6 +149,22 @@ func DecodeIndexKey(key kv.Key) (tableID int64, indexID int64, indexValues []str
return
}

// DecodeIndexKey decodes the key and gets the tableID, indexID, indexValues.
func DecodeIndexKeyPrefix(key kv.Key) (tableID int64, indexID int64, indexValues []byte, err error) {
k := key

tableID, indexID, isRecord, err := DecodeKeyHead(key)
if err != nil {
return 0, 0, nil, errors.Trace(err)
}
if isRecord {
return 0, 0, nil, errInvalidIndexKey.GenWithStack("invalid index key - %q", k)
}
indexValues = key[prefixLen+idLen:]

return tableID, indexID, indexValues, nil
}

// DecodeKeyHead decodes the key's head and gets the tableID, indexID. isRecordKey is true when is a record key.
func DecodeKeyHead(key kv.Key) (tableID int64, indexID int64, isRecordKey bool, err error) {
isRecordKey = false
Expand Down