diff --git a/DEPS.bzl b/DEPS.bzl index d19c145d56fc4..fa7de8864d94c 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -1872,6 +1872,13 @@ def go_deps(): sum = "h1:0Vihzu20St42/UDsvZGdNE6jak7oi/UOeMzwMPHkgFY=", version = "v3.2.0+incompatible", ) + go_repository( + name = "com_github_jarcoal_httpmock", + build_file_proto_mode = "disable", + importpath = "github.com/jarcoal/httpmock", + sum = "h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc=", + version = "v1.2.0", + ) go_repository( name = "com_github_jcmturner_aescts_v2", @@ -2326,6 +2333,14 @@ def go_deps(): sum = "h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=", version = "v1.0.1", ) + go_repository( + name = "com_github_maxatome_go_testdeep", + build_file_proto_mode = "disable", + importpath = "github.com/maxatome/go-testdeep", + sum = "h1:Tgh5efyCYyJFGUYiT0qxBSIDeXw0F5zSoatlou685kk=", + version = "v1.11.0", + ) + go_repository( name = "com_github_mbilski_exhaustivestruct", build_file_proto_mode = "disable", diff --git a/executor/BUILD.bazel b/executor/BUILD.bazel index 49dc2492302b2..2076576bbb8b7 100644 --- a/executor/BUILD.bazel +++ b/executor/BUILD.bazel @@ -415,6 +415,7 @@ go_test( "//util/topsql/state", "@com_github_golang_protobuf//proto", "@com_github_gorilla_mux//:mux", + "@com_github_jarcoal_httpmock//:httpmock", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_fn//:fn", diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 87e61264554a6..e157ac72f93aa 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -3030,14 +3030,17 @@ func (e *TiFlashSystemTableRetriever) initialize(sctx sessionctx.Context, tiflas return nil } +type tiFlashSQLExecuteResponseMetaColumn struct { + Name string `json:"name"` + Type string `json:"type"` +} + +type tiFlashSQLExecuteResponse struct { + Meta []tiFlashSQLExecuteResponseMetaColumn `json:"meta"` + Data [][]interface{} `json:"data"` +} + func (e *TiFlashSystemTableRetriever) dataForTiFlashSystemTables(ctx sessionctx.Context, tidbDatabases string, tidbTables string) ([][]types.Datum, error) { - var columnNames []string //nolint: prealloc - for _, c := range e.outputCols { - if c.Name.O == "TIFLASH_INSTANCE" { - continue - } - columnNames = append(columnNames, c.Name.L) - } maxCount := 1024 targetTable := strings.ToLower(strings.Replace(e.table.Name.O, "TIFLASH", "DT", 1)) var filters []string @@ -3047,12 +3050,11 @@ func (e *TiFlashSystemTableRetriever) dataForTiFlashSystemTables(ctx sessionctx. if len(tidbTables) > 0 { filters = append(filters, fmt.Sprintf("tidb_table IN (%s)", strings.ReplaceAll(tidbTables, "\"", "'"))) } - sql := fmt.Sprintf("SELECT %s FROM system.%s", strings.Join(columnNames, ","), targetTable) + sql := fmt.Sprintf("SELECT * FROM system.%s", targetTable) if len(filters) > 0 { sql = fmt.Sprintf("%s WHERE %s", sql, strings.Join(filters, " AND ")) } sql = fmt.Sprintf("%s LIMIT %d, %d", sql, e.rowIdx, maxCount) - notNumber := "nan" instanceInfo := e.instanceInfos[e.instanceIdx] url := instanceInfo.url req, err := http.NewRequest(http.MethodGet, url, nil) @@ -3061,6 +3063,7 @@ func (e *TiFlashSystemTableRetriever) dataForTiFlashSystemTables(ctx sessionctx. } q := req.URL.Query() q.Add("query", sql) + q.Add("default_format", "JSONCompact") req.URL.RawQuery = q.Encode() resp, err := util.InternalHTTPClient().Do(req) if err != nil { @@ -3071,54 +3074,68 @@ func (e *TiFlashSystemTableRetriever) dataForTiFlashSystemTables(ctx sessionctx. if err != nil { return nil, errors.Trace(err) } - records := strings.Split(string(body), "\n") - rows := make([][]types.Datum, 0, len(records)) - for _, record := range records { - if len(record) == 0 { - continue + var result tiFlashSQLExecuteResponse + err = json.Unmarshal(body, &result) + if err != nil { + return nil, errors.Wrapf(err, "Failed to decode JSON from TiFlash") + } + + // Map result columns back to our columns. It is possible that some columns cannot be + // recognized and some other columns are missing. This may happen during upgrading. + outputColIndexMap := map[string]int{} // Map from TiDB Column name to Output Column Index + for idx, c := range e.outputCols { + outputColIndexMap[c.Name.L] = idx + } + tiflashColIndexMap := map[int]int{} // Map from TiFlash Column index to Output Column Index + for tiFlashColIdx, col := range result.Meta { + if outputIdx, ok := outputColIndexMap[strings.ToLower(col.Name)]; ok { + tiflashColIndexMap[tiFlashColIdx] = outputIdx } - fields := strings.Split(record, "\t") - if len(fields) < len(e.outputCols)-1 { - return nil, errors.Errorf("Record from tiflash doesn't match schema %v", fields) + } + outputRows := make([][]types.Datum, 0, len(result.Data)) + for _, rowFields := range result.Data { + if len(rowFields) == 0 { + continue } - row := make([]types.Datum, len(e.outputCols)) - for index, column := range e.outputCols { - if column.Name.O == "TIFLASH_INSTANCE" { + outputRow := make([]types.Datum, len(e.outputCols)) + for tiFlashColIdx, fieldValue := range rowFields { + outputIdx, ok := tiflashColIndexMap[tiFlashColIdx] + if !ok { + // Discard this field, we don't know which output column is the destination continue } + if fieldValue == nil { + continue + } + valStr := fmt.Sprint(fieldValue) + column := e.outputCols[outputIdx] if column.GetType() == mysql.TypeVarchar { - row[index].SetString(fields[index], mysql.DefaultCollationName) + outputRow[outputIdx].SetString(valStr, mysql.DefaultCollationName) } else if column.GetType() == mysql.TypeLonglong { - if fields[index] == notNumber { - continue - } - value, err := strconv.ParseInt(fields[index], 10, 64) + value, err := strconv.ParseInt(valStr, 10, 64) if err != nil { return nil, errors.Trace(err) } - row[index].SetInt64(value) + outputRow[outputIdx].SetInt64(value) } else if column.GetType() == mysql.TypeDouble { - if fields[index] == notNumber { - continue - } - value, err := strconv.ParseFloat(fields[index], 64) + value, err := strconv.ParseFloat(valStr, 64) if err != nil { return nil, errors.Trace(err) } - row[index].SetFloat64(value) + outputRow[outputIdx].SetFloat64(value) } else { return nil, errors.Errorf("Meet column of unknown type %v", column) } } - row[len(e.outputCols)-1].SetString(instanceInfo.id, mysql.DefaultCollationName) - rows = append(rows, row) + outputRow[len(e.outputCols)-1].SetString(instanceInfo.id, mysql.DefaultCollationName) + outputRows = append(outputRows, outputRow) } - e.rowIdx += len(rows) - if len(rows) < maxCount { + e.rowIdx += len(outputRows) + if len(outputRows) < maxCount { e.instanceIdx += 1 e.rowIdx = 0 } - return rows, nil + return outputRows, nil } func (e *memtableRetriever) setDataForAttributes(ctx sessionctx.Context, is infoschema.InfoSchema) error { diff --git a/executor/infoschema_reader_test.go b/executor/infoschema_reader_test.go index b1c371b77e6d5..dfc4163d93f59 100644 --- a/executor/infoschema_reader_test.go +++ b/executor/infoschema_reader_test.go @@ -16,11 +16,13 @@ package executor_test import ( "fmt" + "os" "strconv" "strings" "testing" "time" + "github.com/jarcoal/httpmock" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/domain/infosync" @@ -649,11 +651,97 @@ func TestSequences(t *testing.T) { tk.MustQuery("SELECT TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME , TABLE_TYPE, ENGINE, TABLE_ROWS FROM information_schema.tables WHERE TABLE_TYPE='SEQUENCE' AND TABLE_NAME='seq2'").Check(testkit.Rows("def test seq2 SEQUENCE InnoDB 1")) } -func TestTiFlashSystemTables(t *testing.T) { +func TestTiFlashSystemTableWithTiFlashV620(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + instances := []string{ + "tiflash,127.0.0.1:3933,127.0.0.1:7777,,", + "tikv,127.0.0.1:11080,127.0.0.1:10080,,", + } + fpName := "github.com/pingcap/tidb/infoschema/mockStoreServerInfo" + fpExpr := `return("` + strings.Join(instances, ";") + `")` + require.NoError(t, failpoint.Enable(fpName, fpExpr)) + defer func() { require.NoError(t, failpoint.Disable(fpName)) }() + + httpmock.RegisterResponder("GET", "http://127.0.0.1:7777/config", + httpmock.NewStringResponder(200, ` +{ + "raftstore-proxy": {}, + "engine-store":{ + "http_port":8123, + "tcp_port":9000 + } +} + `)) + + data, err := os.ReadFile("testdata/tiflash_v620_dt_segments.json") + require.NoError(t, err) + httpmock.RegisterResponder("GET", "http://127.0.0.1:8123?default_format=JSONCompact&query=SELECT+%2A+FROM+system.dt_segments+LIMIT+0%2C+1024", httpmock.NewBytesResponder(200, data)) + + data, err = os.ReadFile("testdata/tiflash_v620_dt_tables.json") + require.NoError(t, err) + httpmock.RegisterResponder("GET", "http://127.0.0.1:8123?default_format=JSONCompact&query=SELECT+%2A+FROM+system.dt_tables+LIMIT+0%2C+1024", httpmock.NewBytesResponder(200, data)) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustQuery("select * from information_schema.TIFLASH_SEGMENTS;").Check(testkit.Rows( + "db_1 t_10 mysql tables_priv 10 0 1 [-9223372036854775808,9223372036854775807) 0 0 0 2032 127.0.0.1:3933", + "db_1 t_8 mysql db 8 0 1 [-9223372036854775808,9223372036854775807) 0 0 0 2032 127.0.0.1:3933", + "db_2 t_70 test segment 70 0 1 [01,FA) 30511 50813627 0.6730359542460096 3578860 409336 127.0.0.1:3933", + )) + tk.MustQuery("show warnings").Check(testkit.Rows()) + + tk.MustQuery("select * from information_schema.TIFLASH_TABLES;").Check(testkit.Rows( + "db_1 t_10 mysql tables_priv 10 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127.0.0.1:3933", + "db_1 t_8 mysql db 8 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127.0.0.1:3933", + "db_2 t_70 test segment 70 0 1 102000 169873868 0 0 0 0 0 102000 169873868 0 0 0 1 102000 169873868 43867622 102000 169873868 0 13 13 7846.153846153846 13067220.615384616 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127.0.0.1:3933", + )) + tk.MustQuery("show warnings").Check(testkit.Rows()) +} + +func TestTiFlashSystemTableWithTiFlashV630(t *testing.T) { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + instances := []string{ + "tiflash,127.0.0.1:3933,127.0.0.1:7777,,", + "tikv,127.0.0.1:11080,127.0.0.1:10080,,", + } + fpName := "github.com/pingcap/tidb/infoschema/mockStoreServerInfo" + fpExpr := `return("` + strings.Join(instances, ";") + `")` + require.NoError(t, failpoint.Enable(fpName, fpExpr)) + defer func() { require.NoError(t, failpoint.Disable(fpName)) }() + + httpmock.RegisterResponder("GET", "http://127.0.0.1:7777/config", + httpmock.NewStringResponder(200, ` +{ + "raftstore-proxy": {}, + "engine-store":{ + "http_port":8123, + "tcp_port":9000 + } +} + `)) + + data, err := os.ReadFile("testdata/tiflash_v630_dt_segments.json") + require.NoError(t, err) + httpmock.RegisterResponder("GET", "http://127.0.0.1:8123?default_format=JSONCompact&query=SELECT+%2A+FROM+system.dt_segments+LIMIT+0%2C+1024", httpmock.NewBytesResponder(200, data)) + store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) - tk.MustExec("select * from information_schema.TIFLASH_TABLES;") - tk.MustExec("select * from information_schema.TIFLASH_SEGMENTS;") + tk.MustQuery("select * from information_schema.TIFLASH_SEGMENTS;").Check(testkit.Rows( + "db_1 t_10 mysql tables_priv 10 0 1 [-9223372036854775808,9223372036854775807) 0 0 0 0 0 0 0 2 0 0 0 0 0 2032 3 0 0 1 1 0 0 0 0 127.0.0.1:3933", + "db_2 t_70 test segment 70 436272981189328904 1 [01,FA) 5 102000 169874232 0 0 0 0 0 2 0 0 0 0 0 2032 3 102000 169874232 1 68 102000 169874232 43951837 20 127.0.0.1:3933", + "db_2 t_75 test segment 75 0 1 [01,013130303030393535FF61653666642D6136FF61382D343032382DFF616436312D663736FF3062323736643461FF3600000000000000F8) 2 0 0 0 0 1 1 110 0 0 4 4 0 2032 111 0 0 1 70 0 0 0 0 127.0.0.1:3933", + "db_2 t_75 test segment 75 0 113 [013130303030393535FF61653666642D6136FF61382D343032382DFF616436312D663736FF3062323736643461FF3600000000000000F8,013139393938363264FF33346535382D3735FF31382D343661612DFF626235392D636264FF3139333434623736FF3100000000000000F9) 2 10167 16932617 0.4887380741615029 0 0 0 0 114 4969 8275782 2 0 0 63992 112 5198 8656835 1 71 5198 8656835 2254100 1 127.0.0.1:3933", + "db_2 t_75 test segment 75 0 116 [013139393938363264FF33346535382D3735FF31382D343661612DFF626235392D636264FF3139333434623736FF3100000000000000F9,013330303131383034FF61323537662D6638FF63302D346466622DFF383235632D353361FF3236306338616662FF3400000000000000F8) 3 8 13322 0.5 3 4986 1 0 117 1 1668 4 3 4986 2032 115 4 6668 1 78 4 6668 6799 1 127.0.0.1:3933", + "db_2 t_75 test segment 75 0 125 [013330303131383034FF61323537662D6638FF63302D346466622DFF383235632D353361FF3236306338616662FF3400000000000000F8,013339393939613861FF30663062332D6537FF32372D346234642DFF396535632D363865FF3336323066383431FF6300000000000000F9) 2 8677 14451079 0.4024432407514118 3492 5816059 3 0 126 0 0 0 0 5816059 2032 124 5185 8635020 1 79 5185 8635020 2247938 1 127.0.0.1:3933", + "db_2 t_75 test segment 75 0 128 [013339393939613861FF30663062332D6537FF32372D346234642DFF396535632D363865FF3336323066383431FF6300000000000000F9,013730303031636230FF32663330652D3539FF62352D346134302DFF613539312D383930FF6132316364633466FF3200000000000000F8) 0 1 1668 1 0 0 0 0 129 1 1668 5 4 0 2032 127 0 0 1 78 4 6668 6799 1 127.0.0.1:3933", + "db_2 t_75 test segment 75 0 119 [013730303031636230FF32663330652D3539FF62352D346134302DFF613539312D383930FF6132316364633466FF3200000000000000F8,013739393939386561FF36393566612D3534FF64302D346437642DFF383136612D646335FF6432613130353533FF3200000000000000F9) 2 10303 17158730 0.489372027564787 0 0 0 0 120 5042 8397126 2 0 0 63992 118 5261 8761604 1 77 5261 8761604 2280506 1 127.0.0.1:3933", + "db_2 t_75 test segment 75 0 122 [013739393939386561FF36393566612D3534FF64302D346437642DFF383136612D646335FF6432613130353533FF3200000000000000F9,FA) 0 1 1663 1 0 0 0 0 123 1 1663 4 3 0 2032 121 0 0 1 78 4 6668 6799 1 127.0.0.1:3933", + )) + tk.MustQuery("show warnings").Check(testkit.Rows()) } func TestTablesPKType(t *testing.T) { diff --git a/executor/testdata/tiflash_v620_dt_segments.json b/executor/testdata/tiflash_v620_dt_segments.json new file mode 100644 index 0000000000000..f2b5ced818899 --- /dev/null +++ b/executor/testdata/tiflash_v620_dt_segments.json @@ -0,0 +1,99 @@ +{ + "meta": + [ + { + "name": "database", + "type": "String" + }, + { + "name": "table", + "type": "String" + }, + { + "name": "tidb_database", + "type": "String" + }, + { + "name": "tidb_table", + "type": "String" + }, + { + "name": "table_id", + "type": "Int64" + }, + { + "name": "is_tombstone", + "type": "UInt64" + }, + { + "name": "segment_id", + "type": "UInt64" + }, + { + "name": "range", + "type": "String" + }, + { + "name": "rows", + "type": "UInt64" + }, + { + "name": "size", + "type": "UInt64" + }, + { + "name": "delete_ranges", + "type": "UInt64" + }, + { + "name": "stable_size_on_disk", + "type": "UInt64" + }, + { + "name": "delta_pack_count", + "type": "UInt64" + }, + { + "name": "stable_pack_count", + "type": "UInt64" + }, + { + "name": "avg_delta_pack_rows", + "type": "Float64" + }, + { + "name": "avg_stable_pack_rows", + "type": "Float64" + }, + { + "name": "delta_rate", + "type": "Float64" + }, + { + "name": "delta_cache_size", + "type": "UInt64" + }, + { + "name": "delta_index_size", + "type": "UInt64" + } + ], + + "data": + [ + ["db_1", "t_10", "mysql", "tables_priv", "10", "0", "1", "[-9223372036854775808,9223372036854775807)", "0", "0", "0", "0", "0", "0", null, null, null, "0", "2032"], + ["db_1", "t_8", "mysql", "db", "8", "0", "1", "[-9223372036854775808,9223372036854775807)", "0", "0", "0", "0", "0", "0", null, null, null, "0", "2032"], + ["db_2", "t_70", "test", "segment", "70", "0", "1", "[01,FA)", "30511", "50813627", "12", "4296273", "18", "2", 1140.8333333333333, 4988, 0.6730359542460096, "3578860", "409336"] + ], + + "rows": 3, + + "rows_before_limit_at_least": 3, + + "statistics": + { + "elapsed": 0.000075, + "rows_read": 3, + "bytes_read": 8324 + } +} diff --git a/executor/testdata/tiflash_v620_dt_tables.json b/executor/testdata/tiflash_v620_dt_tables.json new file mode 100644 index 0000000000000..3fcc00b524373 --- /dev/null +++ b/executor/testdata/tiflash_v620_dt_tables.json @@ -0,0 +1,267 @@ +{ + "meta": + [ + { + "name": "database", + "type": "String" + }, + { + "name": "table", + "type": "String" + }, + { + "name": "tidb_database", + "type": "String" + }, + { + "name": "tidb_table", + "type": "String" + }, + { + "name": "table_id", + "type": "Int64" + }, + { + "name": "is_tombstone", + "type": "UInt64" + }, + { + "name": "segment_count", + "type": "UInt64" + }, + { + "name": "total_rows", + "type": "UInt64" + }, + { + "name": "total_size", + "type": "UInt64" + }, + { + "name": "total_delete_ranges", + "type": "UInt64" + }, + { + "name": "delta_rate_rows", + "type": "Float64" + }, + { + "name": "delta_rate_segments", + "type": "Float64" + }, + { + "name": "delta_placed_rate", + "type": "Float64" + }, + { + "name": "delta_cache_size", + "type": "UInt64" + }, + { + "name": "delta_cache_rate", + "type": "Float64" + }, + { + "name": "delta_cache_wasted_rate", + "type": "Float64" + }, + { + "name": "delta_index_size", + "type": "UInt64" + }, + { + "name": "avg_segment_rows", + "type": "Float64" + }, + { + "name": "avg_segment_size", + "type": "Float64" + }, + { + "name": "delta_count", + "type": "UInt64" + }, + { + "name": "total_delta_rows", + "type": "UInt64" + }, + { + "name": "total_delta_size", + "type": "UInt64" + }, + { + "name": "avg_delta_rows", + "type": "Float64" + }, + { + "name": "avg_delta_size", + "type": "Float64" + }, + { + "name": "avg_delta_delete_ranges", + "type": "Float64" + }, + { + "name": "stable_count", + "type": "UInt64" + }, + { + "name": "total_stable_rows", + "type": "UInt64" + }, + { + "name": "total_stable_size", + "type": "UInt64" + }, + { + "name": "total_stable_size_on_disk", + "type": "UInt64" + }, + { + "name": "avg_stable_rows", + "type": "Float64" + }, + { + "name": "avg_stable_size", + "type": "Float64" + }, + { + "name": "total_pack_count_in_delta", + "type": "UInt64" + }, + { + "name": "avg_pack_count_in_delta", + "type": "Float64" + }, + { + "name": "avg_pack_rows_in_delta", + "type": "Float64" + }, + { + "name": "avg_pack_size_in_delta", + "type": "Float64" + }, + { + "name": "total_pack_count_in_stable", + "type": "UInt64" + }, + { + "name": "avg_pack_count_in_stable", + "type": "Float64" + }, + { + "name": "avg_pack_rows_in_stable", + "type": "Float64" + }, + { + "name": "avg_pack_size_in_stable", + "type": "Float64" + }, + { + "name": "storage_stable_num_snapshots", + "type": "UInt64" + }, + { + "name": "storage_stable_oldest_snapshot_lifetime", + "type": "Float64" + }, + { + "name": "storage_stable_oldest_snapshot_thread_id", + "type": "UInt64" + }, + { + "name": "storage_stable_oldest_snapshot_tracing_id", + "type": "String" + }, + { + "name": "storage_stable_num_pages", + "type": "UInt64" + }, + { + "name": "storage_stable_num_normal_pages", + "type": "UInt64" + }, + { + "name": "storage_stable_max_page_id", + "type": "UInt64" + }, + { + "name": "storage_delta_num_snapshots", + "type": "UInt64" + }, + { + "name": "storage_delta_oldest_snapshot_lifetime", + "type": "Float64" + }, + { + "name": "storage_delta_oldest_snapshot_thread_id", + "type": "UInt64" + }, + { + "name": "storage_delta_oldest_snapshot_tracing_id", + "type": "String" + }, + { + "name": "storage_delta_num_pages", + "type": "UInt64" + }, + { + "name": "storage_delta_num_normal_pages", + "type": "UInt64" + }, + { + "name": "storage_delta_max_page_id", + "type": "UInt64" + }, + { + "name": "storage_meta_num_snapshots", + "type": "UInt64" + }, + { + "name": "storage_meta_oldest_snapshot_lifetime", + "type": "Float64" + }, + { + "name": "storage_meta_oldest_snapshot_thread_id", + "type": "UInt64" + }, + { + "name": "storage_meta_oldest_snapshot_tracing_id", + "type": "String" + }, + { + "name": "storage_meta_num_pages", + "type": "UInt64" + }, + { + "name": "storage_meta_num_normal_pages", + "type": "UInt64" + }, + { + "name": "storage_meta_max_page_id", + "type": "UInt64" + }, + { + "name": "background_tasks_length", + "type": "UInt64" + } + ], + + "data": + [ + ["db_1", "t_10", "mysql", "tables_priv", "10", "0", "1", "0", "0", "0", null, 0, null, "0", null, null, "0", 0, 0, "0", "0", "0", null, null, null, "0", "0", "0", "0", null, null, "0", null, null, null, "0", null, null, null, "0", 0, "0", "", "0", "0", "0", "0", 0, "0", "", "0", "0", "0", "0", 0, "0", "", "0", "0", "0", "0"], + ["db_1", "t_8", "mysql", "db", "8", "0", "1", "0", "0", "0", null, 0, null, "0", null, null, "0", 0, 0, "0", "0", "0", null, null, null, "0", "0", "0", "0", null, null, "0", null, null, null, "0", null, null, null, "0", 0, "0", "", "0", "0", "0", "0", 0, "0", "", "0", "0", "0", "0", 0, "0", "", "0", "0", "0", "0"], + ["db_2", "t_70", "test", "segment", "70", "0", "1", "102000", "169873868", "0", 0, 0, null, "0", null, null, "0", 102000, 169873868, "0", "0", "0", null, null, null, "1", "102000", "169873868", "43867622", 102000, 169873868, "0", null, null, null, "13", 13, 7846.153846153846, 13067220.615384616, "0", 0, "0", "", "0", "0", "0", "0", 0, "0", "", "0", "0", "0", "0", 0, "0", "", "0", "0", "0", "0"] + ], + + "rows": 3, + + "rows_before_limit_at_least": 3, + + "statistics": + { + "elapsed": 0.000217, + "rows_read": 3, + "bytes_read": 19346 + } +} diff --git a/executor/testdata/tiflash_v630_dt_segments.json b/executor/testdata/tiflash_v630_dt_segments.json new file mode 100644 index 0000000000000..9c1da2b4bb47b --- /dev/null +++ b/executor/testdata/tiflash_v630_dt_segments.json @@ -0,0 +1,157 @@ +{ + "meta": + [ + { + "name": "database", + "type": "String" + }, + { + "name": "table", + "type": "String" + }, + { + "name": "tidb_database", + "type": "String" + }, + { + "name": "tidb_table", + "type": "String" + }, + { + "name": "table_id", + "type": "Int64" + }, + { + "name": "is_tombstone", + "type": "UInt64" + }, + { + "name": "segment_id", + "type": "UInt64" + }, + { + "name": "range", + "type": "String" + }, + { + "name": "epoch", + "type": "UInt64" + }, + { + "name": "rows", + "type": "UInt64" + }, + { + "name": "size", + "type": "UInt64" + }, + { + "name": "delta_rate", + "type": "Float64" + }, + { + "name": "delta_memtable_rows", + "type": "UInt64" + }, + { + "name": "delta_memtable_size", + "type": "UInt64" + }, + { + "name": "delta_memtable_column_files", + "type": "UInt64" + }, + { + "name": "delta_memtable_delete_ranges", + "type": "UInt64" + }, + { + "name": "delta_persisted_page_id", + "type": "UInt64" + }, + { + "name": "delta_persisted_rows", + "type": "UInt64" + }, + { + "name": "delta_persisted_size", + "type": "UInt64" + }, + { + "name": "delta_persisted_column_files", + "type": "UInt64" + }, + { + "name": "delta_persisted_delete_ranges", + "type": "UInt64" + }, + { + "name": "delta_cache_size", + "type": "UInt64" + }, + { + "name": "delta_index_size", + "type": "UInt64" + }, + { + "name": "stable_page_id", + "type": "UInt64" + }, + { + "name": "stable_rows", + "type": "UInt64" + }, + { + "name": "stable_size", + "type": "UInt64" + }, + { + "name": "stable_dmfiles", + "type": "UInt64" + }, + { + "name": "stable_dmfiles_id_0", + "type": "UInt64" + }, + { + "name": "stable_dmfiles_rows", + "type": "UInt64" + }, + { + "name": "stable_dmfiles_size", + "type": "UInt64" + }, + { + "name": "stable_dmfiles_size_on_disk", + "type": "UInt64" + }, + { + "name": "stable_dmfiles_packs", + "type": "UInt64" + } + ], + + "data": + [ + ["db_1", "t_10", "mysql", "tables_priv", "10", "0", "1", "[-9223372036854775808,9223372036854775807)", "0", "0", "0", null, "0", "0", "0", "0", "2", "0", "0", "0", "0", "0", "2032", "3", "0", "0", "1", "1", "0", "0", "0", "0"], + ["db_2", "t_70", "test", "segment", "70", "436272981189328904", "1", "[01,FA)", "5", "102000", "169874232", 0, "0", "0", "0", "0", "2", "0", "0", "0", "0", "0", "2032", "3", "102000", "169874232", "1", "68", "102000", "169874232", "43951837", "20"], + ["db_2", "t_75", "test", "segment", "75", "0", "1", "[01,013130303030393535FF61653666642D6136FF61382D343032382DFF616436312D663736FF3062323736643461FF3600000000000000F8)", "2", "0", "0", null, "0", "0", "1", "1", "110", "0", "0", "4", "4", "0", "2032", "111", "0", "0", "1", "70", "0", "0", "0", "0"], + ["db_2", "t_75", "test", "segment", "75", "0", "113", "[013130303030393535FF61653666642D6136FF61382D343032382DFF616436312D663736FF3062323736643461FF3600000000000000F8,013139393938363264FF33346535382D3735FF31382D343661612DFF626235392D636264FF3139333434623736FF3100000000000000F9)", "2", "10167", "16932617", 0.4887380741615029, "0", "0", "0", "0", "114", "4969", "8275782", "2", "0", "0", "63992", "112", "5198", "8656835", "1", "71", "5198", "8656835", "2254100", "1"], + ["db_2", "t_75", "test", "segment", "75", "0", "116", "[013139393938363264FF33346535382D3735FF31382D343661612DFF626235392D636264FF3139333434623736FF3100000000000000F9,013330303131383034FF61323537662D6638FF63302D346466622DFF383235632D353361FF3236306338616662FF3400000000000000F8)", "3", "8", "13322", 0.5, "3", "4986", "1", "0", "117", "1", "1668", "4", "3", "4986", "2032", "115", "4", "6668", "1", "78", "4", "6668", "6799", "1"], + ["db_2", "t_75", "test", "segment", "75", "0", "125", "[013330303131383034FF61323537662D6638FF63302D346466622DFF383235632D353361FF3236306338616662FF3400000000000000F8,013339393939613861FF30663062332D6537FF32372D346234642DFF396535632D363865FF3336323066383431FF6300000000000000F9)", "2", "8677", "14451079", 0.4024432407514118, "3492", "5816059", "3", "0", "126", "0", "0", "0", "0", "5816059", "2032", "124", "5185", "8635020", "1", "79", "5185", "8635020", "2247938", "1"], + ["db_2", "t_75", "test", "segment", "75", "0", "128", "[013339393939613861FF30663062332D6537FF32372D346234642DFF396535632D363865FF3336323066383431FF6300000000000000F9,013730303031636230FF32663330652D3539FF62352D346134302DFF613539312D383930FF6132316364633466FF3200000000000000F8)", "0", "1", "1668", 1, "0", "0", "0", "0", "129", "1", "1668", "5", "4", "0", "2032", "127", "0", "0", "1", "78", "4", "6668", "6799", "1"], + ["db_2", "t_75", "test", "segment", "75", "0", "119", "[013730303031636230FF32663330652D3539FF62352D346134302DFF613539312D383930FF6132316364633466FF3200000000000000F8,013739393939386561FF36393566612D3534FF64302D346437642DFF383136612D646335FF6432613130353533FF3200000000000000F9)", "2", "10303", "17158730", 0.489372027564787, "0", "0", "0", "0", "120", "5042", "8397126", "2", "0", "0", "63992", "118", "5261", "8761604", "1", "77", "5261", "8761604", "2280506", "1"], + ["db_2", "t_75", "test", "segment", "75", "0", "122", "[013739393939386561FF36393566612D3534FF64302D346437642DFF383136612D646335FF6432613130353533FF3200000000000000F9,FA)", "0", "1", "1663", 1, "0", "0", "0", "0", "123", "1", "1663", "4", "3", "0", "2032", "121", "0", "0", "1", "78", "4", "6668", "6799", "1"] + ], + + "rows": 9, + + "rows_before_limit_at_least": 9, + + "statistics": + { + "elapsed": 0.000759, + "rows_read": 9, + "bytes_read": 15477 + } +} diff --git a/go.mod b/go.mod index 0b896256bc44d..caf35784115a0 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/gostaticanalysis/forcetypeassert v0.1.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/iancoleman/strcase v0.2.0 + github.com/jarcoal/httpmock v1.2.0 github.com/jedib0t/go-pretty/v6 v6.2.2 github.com/jingyugao/rowserrcheck v1.1.1 github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df diff --git a/go.sum b/go.sum index bc8d73264874d..036326aff8a42 100644 --- a/go.sum +++ b/go.sum @@ -535,6 +535,8 @@ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= +github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -650,6 +652,7 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/maxatome/go-testdeep v1.11.0 h1:Tgh5efyCYyJFGUYiT0qxBSIDeXw0F5zSoatlou685kk= github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgechev/revive v1.2.4-0.20220827111817-553604eaced5 h1:a+itKsYpxka50MyaWQW1XCZ1vwfgjzVy/OzZ7DC/4+U= diff --git a/infoschema/tables.go b/infoschema/tables.go index 209bed56cca62..74d064b55759a 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -1368,18 +1368,21 @@ var tableTableTiFlashTablesCols = []columnInfo{ {name: "STORAGE_STABLE_NUM_SNAPSHOTS", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_STABLE_OLDEST_SNAPSHOT_LIFETIME", tp: mysql.TypeDouble, size: 64}, {name: "STORAGE_STABLE_OLDEST_SNAPSHOT_THREAD_ID", tp: mysql.TypeLonglong, size: 64}, + {name: "STORAGE_STABLE_OLDEST_SNAPSHOT_TRACING_ID", tp: mysql.TypeVarchar, size: 128}, {name: "STORAGE_STABLE_NUM_PAGES", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_STABLE_NUM_NORMAL_PAGES", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_STABLE_MAX_PAGE_ID", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_DELTA_NUM_SNAPSHOTS", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_DELTA_OLDEST_SNAPSHOT_LIFETIME", tp: mysql.TypeDouble, size: 64}, {name: "STORAGE_DELTA_OLDEST_SNAPSHOT_THREAD_ID", tp: mysql.TypeLonglong, size: 64}, + {name: "STORAGE_DELTA_OLDEST_SNAPSHOT_TRACING_ID", tp: mysql.TypeVarchar, size: 128}, {name: "STORAGE_DELTA_NUM_PAGES", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_DELTA_NUM_NORMAL_PAGES", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_DELTA_MAX_PAGE_ID", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_META_NUM_SNAPSHOTS", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_META_OLDEST_SNAPSHOT_LIFETIME", tp: mysql.TypeDouble, size: 64}, {name: "STORAGE_META_OLDEST_SNAPSHOT_THREAD_ID", tp: mysql.TypeLonglong, size: 64}, + {name: "STORAGE_META_OLDEST_SNAPSHOT_TRACING_ID", tp: mysql.TypeVarchar, size: 128}, {name: "STORAGE_META_NUM_PAGES", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_META_NUM_NORMAL_PAGES", tp: mysql.TypeLonglong, size: 64}, {name: "STORAGE_META_MAX_PAGE_ID", tp: mysql.TypeLonglong, size: 64}, @@ -1396,17 +1399,30 @@ var tableTableTiFlashSegmentsCols = []columnInfo{ {name: "IS_TOMBSTONE", tp: mysql.TypeLonglong, size: 64}, {name: "SEGMENT_ID", tp: mysql.TypeLonglong, size: 64}, {name: "RANGE", tp: mysql.TypeVarchar, size: 64}, + {name: "EPOCH", tp: mysql.TypeLonglong, size: 64}, {name: "ROWS", tp: mysql.TypeLonglong, size: 64}, {name: "SIZE", tp: mysql.TypeLonglong, size: 64}, - {name: "DELETE_RANGES", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_SIZE_ON_DISK", tp: mysql.TypeLonglong, size: 64}, - {name: "DELTA_PACK_COUNT", tp: mysql.TypeLonglong, size: 64}, - {name: "STABLE_PACK_COUNT", tp: mysql.TypeLonglong, size: 64}, - {name: "AVG_DELTA_PACK_ROWS", tp: mysql.TypeDouble, size: 64}, - {name: "AVG_STABLE_PACK_ROWS", tp: mysql.TypeDouble, size: 64}, {name: "DELTA_RATE", tp: mysql.TypeDouble, size: 64}, + {name: "DELTA_MEMTABLE_ROWS", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_MEMTABLE_SIZE", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_MEMTABLE_COLUMN_FILES", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_MEMTABLE_DELETE_RANGES", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_PERSISTED_PAGE_ID", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_PERSISTED_ROWS", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_PERSISTED_SIZE", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_PERSISTED_COLUMN_FILES", tp: mysql.TypeLonglong, size: 64}, + {name: "DELTA_PERSISTED_DELETE_RANGES", tp: mysql.TypeLonglong, size: 64}, {name: "DELTA_CACHE_SIZE", tp: mysql.TypeLonglong, size: 64}, {name: "DELTA_INDEX_SIZE", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_PAGE_ID", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_ROWS", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_SIZE", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_DMFILES", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_DMFILES_ID_0", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_DMFILES_ROWS", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_DMFILES_SIZE", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_DMFILES_SIZE_ON_DISK", tp: mysql.TypeLonglong, size: 64}, + {name: "STABLE_DMFILES_PACKS", tp: mysql.TypeLonglong, size: 64}, {name: "TIFLASH_INSTANCE", tp: mysql.TypeVarchar, size: 64}, } @@ -1774,6 +1790,24 @@ func GetPDServerInfo(ctx sessionctx.Context) ([]ServerInfo, error) { // GetStoreServerInfo returns all store nodes(TiKV or TiFlash) cluster information func GetStoreServerInfo(ctx sessionctx.Context) ([]ServerInfo, error) { + failpoint.Inject("mockStoreServerInfo", func(val failpoint.Value) { + if s := val.(string); len(s) > 0 { + var servers []ServerInfo + for _, server := range strings.Split(s, ";") { + parts := strings.Split(server, ",") + servers = append(servers, ServerInfo{ + ServerType: parts[0], + Address: parts[1], + StatusAddr: parts[2], + Version: parts[3], + GitHash: parts[4], + StartTimestamp: 0, + }) + } + failpoint.Return(servers, nil) + } + }) + isTiFlashStore := func(store *metapb.Store) bool { isTiFlash := false for _, label := range store.Labels {